diff --git a/.gitlab/stages/01.tests.yml b/.gitlab/stages/01.tests.yml
index 44ded54afd..25d9d286fc 100644
--- a/.gitlab/stages/01.tests.yml
+++ b/.gitlab/stages/01.tests.yml
@@ -10,7 +10,7 @@ include:
   - template: Security/Secret-Detection.gitlab-ci.yml
 
 build_image:
-  tags: 
+  tags:
     - ${TAG}
   image: docker:26.1.4-dind
   timeout: 45m
@@ -90,7 +90,7 @@ unit_tests:
   parallel:
     matrix:
       - TAG: latest
-      - TAG: f2d356582247e1df5a4c0f7c426d33096a394dc1
+      - TAG: f6ee2ebaf2c8a3bfa091a8327452078ecd89fc3a
   tags: [8xL40S]
   variables:
     GIT_STRATEGY: clone
@@ -164,4 +164,4 @@ secret_detection:
         echo "Atleast one vulnerability has been found"
         cat gl-secret-detection-report.json | jq '.'
         exit 1
-      fi
\ No newline at end of file
+      fi
diff --git a/megatron/core/distributed/__init__.py b/megatron/core/distributed/__init__.py
index b375e37376..8264015909 100644
--- a/megatron/core/distributed/__init__.py
+++ b/megatron/core/distributed/__init__.py
@@ -3,4 +3,4 @@
 from .distributed_data_parallel import DistributedDataParallel
 from .distributed_data_parallel_config import DistributedDataParallelConfig
 from .finalize_model_grads import finalize_model_grads
-from .param_and_grad_buffer import ParamAndGradBuffer, shard_buffer
+from .param_and_grad_buffer import ParamAndGradBuffer, partition_buckets, shard_buffer
diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py
index 0451a6e4fb..1c2011d3c6 100644
--- a/megatron/core/distributed/distributed_data_parallel.py
+++ b/megatron/core/distributed/distributed_data_parallel.py
@@ -2,7 +2,7 @@
 
 import logging
 from contextlib import contextmanager
-from typing import Dict, Optional
+from typing import Dict
 
 import torch
 
@@ -10,9 +10,9 @@
 from ..config_logger import has_config_logger_enabled, log_config_to_disk
 from ..transformer.module import MegatronModule
 from ..transformer.transformer_config import TransformerConfig
-from ..utils import log_single_rank
+from ..utils import is_float8tensor, log_single_rank
 from .distributed_data_parallel_config import DistributedDataParallelConfig
-from .param_and_grad_buffer import ParamAndGradBuffer
+from .param_and_grad_buffer import BucketGroup, ParamAndGradBuffer, partition_buckets
 
 logger = logging.getLogger(__name__)
 
@@ -78,7 +78,7 @@ def __init__(
             self.bucket_size = None
 
         self.module = module
-        self.param_to_buffer = {}
+        self.param_to_bucket_group = {}
 
         # Group parameters by their gradient type.
         param_to_name = {}
@@ -100,6 +100,8 @@ def allocate_buffers_for_parameters(
             input_params, data_parallel_group, gradient_scaling_factor
         ):
             param_and_grad_dtype_to_params = {}
+            param_and_grad_dtype_to_offsets = {}
+            param_and_grad_dtype_to_indices = {}
 
             # Group parameters by their gradient type.
             for param in input_params:
@@ -107,12 +109,41 @@ def allocate_buffers_for_parameters(
                     continue
 
                 param_dtype = param.dtype
+                if is_float8tensor(param):
+                    # Currently TE's Float8Tensor is a wrapper of torch.Tensor. It has a "fake"
+                    # dtype (usually a higher precision dtype such as bfloat16), but its actual
+                    # data is stored in the form of a torch uint8 tensor within the Float8Tensor's
+                    # ".data" attribute. Therefore, when creating the param buffer for fp8 params,
+                    # it is necessary to use torch.uint8, not the "fake" dtype got from
+                    # "param.dtype".
+                    param_dtype = torch.uint8
                 grad_dtype = torch.float if self.ddp_config.grad_reduce_in_fp32 else param.dtype
 
                 params = param_and_grad_dtype_to_params.get((param_dtype, grad_dtype), [])
                 params.append(param)
                 param_and_grad_dtype_to_params[(param_dtype, grad_dtype)] = params
 
+                # Get the index of each param among the params with same dtype, if a param is fp8,
+                # use its "fake" high precision dtype to find which params have same dtype with it.
+                # For example:
+                #     Case 1:
+                #         params = [p1(bf16), p2(bf16), p3(bf16), p4(bf16)]
+                #         param_and_grad_dtype_to_indices = {
+                #             (torch.bfloat16, torch.float32): [0, 1, 2, 3],
+                #         }
+                #     Case 2:
+                #         params = [p1(bf16), p2(fp8), p3(fp8), p4(bf16)]
+                #         param_and_grad_dtype_to_indices = {
+                #             (torch.bfloat16, torch.float32): [0, 3],
+                #             (torch.uint8, torch.float32): [1, 2],
+                #         }
+                # We need these indices to load a non-native-fp8 checkpoint in native-fp8 mode.
+                offset = param_and_grad_dtype_to_offsets.get((param.dtype, grad_dtype), 0)
+                param_and_grad_dtype_to_offsets[(param.dtype, grad_dtype)] = offset + 1
+                indices = param_and_grad_dtype_to_indices.get((param_dtype, grad_dtype), [])
+                indices.append(offset)
+                param_and_grad_dtype_to_indices[(param_dtype, grad_dtype)] = indices
+
             if not config.calculate_per_token_loss:
                 target_gradient_scaling_factor = 1.0 / parallel_state.get_data_parallel_world_size()
                 if self.ddp_config.average_in_collective:
@@ -138,12 +169,26 @@ def allocate_buffers_for_parameters(
                         self.bucket_size,
                         param_to_name,
                         gradient_scaling_factor,
+                        param_and_grad_dtype_to_indices[(param_dtype, grad_dtype)],
                     )
                 )
-                for param in params:
-                    self.param_to_buffer[param] = buffers[-1]
 
-            return buffers
+            # In some scenarios, we want to put buckets from different buffers into a group so that
+            # their communication can be aggregated. For example, when there are both fp8 buffers
+            # and bf16 buffers in the model and vpp is enabled, each model chunk will have an fp8
+            # bucket and a bf16 bucket, which doubles the number of communication kernels, and
+            # because of the use of CUDA_DEVICE_MAX_CONNECTIONS=1, having multiple back-to-back
+            # communications will prevent the overlap of the communication kernels with computation
+            # kernels.
+            bucket_groups = partition_buckets(buffers)
+
+            # Create map from param to BucketGroup, used in pre_hook.
+            for bucket_group in bucket_groups:
+                for bucket in bucket_group.buckets:
+                    for param in bucket.params_list:
+                        self.param_to_bucket_group[param] = bucket_group
+
+            return buffers, bucket_groups
 
         if config.calculate_per_token_loss:
             gradient_scaling_factor = 1.0
@@ -160,17 +205,19 @@ def allocate_buffers_for_parameters(
                 expert_gradient_scaling_factor = 1.0 / data_parallel_world_size
 
         # Allocate the param+grad buffers for dense params' grads.
-        self.buffers = allocate_buffers_for_parameters(
+        self.buffers, self.bucket_groups = allocate_buffers_for_parameters(
             dense_params,
             parallel_state.get_data_parallel_group(with_context_parallel=True),
             gradient_scaling_factor=gradient_scaling_factor,
         )
 
         # Allocate separate param+grad buffers for expert parallel params' grads.
-        self.expert_parallel_buffers = allocate_buffers_for_parameters(
-            expert_parallel_params,
-            parallel_state.get_data_modulo_expert_parallel_group(with_context_parallel=True),
-            gradient_scaling_factor=expert_gradient_scaling_factor,
+        self.expert_parallel_buffers, self.expert_parallel_bucket_groups = (
+            allocate_buffers_for_parameters(
+                expert_parallel_params,
+                parallel_state.get_data_modulo_expert_parallel_group(with_context_parallel=True),
+                gradient_scaling_factor=expert_gradient_scaling_factor,
+            )
         )
 
         # Delete references to weight_tensor if they exist since we don't want two parameter copies
@@ -196,7 +243,7 @@ def unmap_weight_tensor(m):
                 param_tmp = param.expand_as(param)
                 # Get the gradient accumulator function.
                 grad_acc = param_tmp.grad_fn.next_functions[0][0]
-                grad_acc.register_hook(self._make_param_hook(param, self.param_to_buffer))
+                grad_acc.register_hook(self._make_param_hook(param, self.param_to_bucket_group))
                 self.grad_accs.append(grad_acc)
 
     def forward(self, *inputs, **kwargs):
@@ -208,7 +255,7 @@ def forward(self, *inputs, **kwargs):
     def _make_param_hook(
         self,
         param: torch.nn.Parameter,
-        param_to_buffer: Dict[torch.nn.Parameter, ParamAndGradBuffer],
+        param_to_bucket_group: Dict[torch.nn.Parameter, BucketGroup],
     ):
         """
         Creates the all-reduce / reduce-scatter hook for backprop.
@@ -227,7 +274,7 @@ def param_hook(*unused):
                 param.grad = None
 
                 if self.ddp_config.overlap_grad_reduce:
-                    param_to_buffer[param].register_grad_ready(param)
+                    param_to_bucket_group[param].register_grad_ready(param)
 
         return param_hook
 
@@ -236,13 +283,13 @@ def no_sync(self):
         """
         Context manager that turns off gradient synchronization.
         """
-        for buffer in self.buffers + self.expert_parallel_buffers:
-            buffer.is_last_microbatch = False
+        for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups:
+            bucket_group.is_last_microbatch = False
         try:
             yield
         finally:
-            for buffer in self.buffers + self.expert_parallel_buffers:
-                buffer.is_last_microbatch = True
+            for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups:
+                bucket_group.is_last_microbatch = True
 
     def start_grad_sync(self, *unused):
         """
@@ -253,8 +300,8 @@ def start_grad_sync(self, *unused):
         calls. When overlap_grad_reduce is set to False, calls synchronous
         communication ops.
         """
-        for buffer in self.buffers + self.expert_parallel_buffers:
-            buffer.start_grad_sync()
+        for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups:
+            bucket_group.start_grad_sync()
 
     def scale_gradients(self, scaling_factor: float) -> None:
         """Scale all gradients inside the buffers by `scaling_factor`."""
@@ -270,8 +317,8 @@ def finish_grad_sync(self):
         calls to complete. When overlap_grad_reduce is set to False, calls synchronous
         communication ops.
         """
-        for buffer in self.buffers + self.expert_parallel_buffers:
-            buffer.finish_grad_sync()
+        for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups:
+            bucket_group.finish_grad_sync()
 
     def zero_grad_buffer(self):
         """
@@ -283,6 +330,8 @@ def zero_grad_buffer(self):
                 param.grad_added_to_main_grad = False
         for buffer in self.buffers + self.expert_parallel_buffers:
             buffer.reset()
+        for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups:
+            bucket_group.reset()
 
     def broadcast_params(self):
         """
diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py
index c1396e0f00..b47be4b75f 100644
--- a/megatron/core/distributed/distributed_data_parallel_config.py
+++ b/megatron/core/distributed/distributed_data_parallel_config.py
@@ -30,3 +30,7 @@ class DistributedDataParallelConfig:
     average_in_collective: bool = False
     """If true, compute average in collective directly, as opposed to dividing by the
        dp_size first and then computing sum in the collective."""
+
+    fp8_param_gather: bool = False
+    """If true, keep the compute param in fp8 (do not use any other intermediate dtype) and
+       perform the param all-gather in fp8."""
diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py
index 77ecd7be25..da238e4306 100644
--- a/megatron/core/distributed/param_and_grad_buffer.py
+++ b/megatron/core/distributed/param_and_grad_buffer.py
@@ -7,8 +7,9 @@
 from typing import Dict, List, Optional
 
 import torch
+from torch.distributed import _coalescing_manager
 
-from ..utils import log_on_each_pipeline_stage
+from ..utils import is_float8tensor, log_on_each_pipeline_stage
 from .distributed_data_parallel_config import DistributedDataParallelConfig
 
 logger = logging.getLogger(__name__)
@@ -37,19 +38,14 @@ def shard_buffer(buffer: torch.Tensor, data_parallel_world_size: int):
 
 class Bucket:
     """
-    Bucket to keep track of a subset of the model's gradients. Provides functionality to register
-    when params in the bucket have grads ready to be synced; an asynchronous communication call
-    is automatically launched when _all_ params in the bucket have grads ready.
+    Bucket to keep track of a subset of the model's parameters and gradients.
 
     Args:
-        ddp_config: DistributedDataParallel config object.
         params: List of parameters whose gradients are collated in this bucket.
         param_data: View in ParamAndGradBuffer.param_data that this bucket is responsible for.
         grad_data: View in ParamAndGradBuffer.grad_data that this bucket is responsible for.
         offset: Offset of this bucket's view in the larger ParamAndGradBuffer.
         numel_unpadded: Number of unpadded elements in bucket.
-        data_parallel_group: Data-parallel process group.
-        data_parallel_world_size: World size using the data-parallel group group.
         gradient_scaling_factor: This factor is utilized to scale gradients prior to their
             communication. Its application is twofold: it facilitates the averaging of gradients
             and the scaling of gradients in the context of the Mixture of Experts (MoE) model.
@@ -57,99 +53,150 @@ class Bucket:
 
     def __init__(
         self,
-        ddp_config: DistributedDataParallelConfig,
         params: List[torch.nn.Parameter],
         param_data: Optional[torch.Tensor],
         grad_data: torch.Tensor,
         offset: int,
         numel_unpadded: int,
-        data_parallel_group: torch.distributed.ProcessGroup,
-        data_parallel_world_size: int,
         gradient_scaling_factor: float,
     ):
-        self.ddp_config = ddp_config
-
-        # State for bookkeeping: params is the set of parameters this bucket is
-        # responsible for, params_with_grad is the set of parameters with grads
-        # available. When overlap_grad_reduce is True, communication (all-reduce
-        # or reduce-scatter) is issued when params_with_grad equals params.
         self.params_list = params
         self.params = set(params)
-        self.params_with_grad = set()
+        # Make sure there are no duplicate params.
+        assert len(self.params_list) == len(self.params)
         self.param_data = param_data
         self.grad_data = grad_data
         # The distributed optimizer needs to keep track of this bucket's offset
         # within the full grad_buffer.
         self.offset = offset
         self.numel_unpadded = numel_unpadded
+        self.gradient_scaling_factor = gradient_scaling_factor
+
+
+class BucketGroup:
+    """
+    Put multiple buckets into a group so that their communications can be aggregated together.
+    Provides functionality to register when params in the bucket group have grads ready to be
+    synced; an asynchronous communication call is automatically launched when _all_ params in
+    the bucket group have grads ready.
+
+    Args:
+        buckets: A list of buckets.
+        ddp_config: DistributedDataParallel config object.
+        data_parallel_group: Data-parallel process group.
+        data_parallel_world_size: World size using the data-parallel group group.
+    """
+
+    def __init__(
+        self,
+        buckets: List[Bucket],
+        ddp_config: DistributedDataParallelConfig,
+        data_parallel_group: torch.distributed.ProcessGroup,
+        data_parallel_world_size: int,
+    ):
+        self.buckets = buckets
+        self.ddp_config = ddp_config
         self.data_parallel_group = data_parallel_group
         self.data_parallel_world_size = data_parallel_world_size
         self.data_parallel_rank = torch.distributed.get_rank(group=data_parallel_group)
-        self.gradient_scaling_factor = gradient_scaling_factor
+
+        # State for bookkeeping: params is the set of parameters this bucket group is
+        # responsible for, params_with_grad is the set of parameters with grads
+        # available. When overlap_grad_reduce is True, communication (all-reduce
+        # or reduce-scatter) is issued when params_with_grad equals params.
+        self.param_to_bucket = {}
+        self.params = set()
+        for bucket in self.buckets:
+            for param in bucket.params_list:
+                self.param_to_bucket[param] = bucket
+                self.params.add(param)
 
         self.reset()
 
     def reset(self):
         """
-        Reset metadata in bucket in preparation for the next iteration of training.
+        Reset metadata in bucket group in preparation for the next iteration of training.
         """
         self.params_with_grad = set()
         self.communication_handle = None
         self.is_communication_outstanding = False
+        self.is_last_microbatch = True
+
+    def check_for_nan_in_grad(self):
+        """
+        Make sure norm of grads in bucket are not NaN prior to data-parallel
+        all-reduce / reduce-scatter.
+        """
+        global_rank = torch.distributed.get_rank()
+        norm_is_nan = self.buckets[0].grad_data.norm(p=2).isnan()
+        for i in range(1, len(self.buckets)):
+            norm_is_nan.logical_or_(self.buckets[i].grad_data.norm(p=2).isnan())
+        assert not norm_is_nan, (
+            f'Rank {global_rank}: found NaN in local grad norm in '
+            f'backward pass before data-parallel communication collective. '
+            f'Device: {torch.cuda.current_device()}, node: {os.uname()[1]}'
+        )
 
     def start_grad_sync(self):
         """
-        Initiates grad sync (all-reduce or reduce-scatter) communication operation
-        for this bucket.
+        Initiates grad sync (all-reduce or reduce-scatter) communication operations
+        for all buckets in the bucket group.
 
-        When overlap_grad_reduce is set to True, dispatches an asynchronous
-        communication call. When overlap_grad_reduce is set to False, makes
-        synchronous call.
+        When overlap_grad_reduce is set to True, dispatches asynchronous communication
+        calls. When overlap_grad_reduce is set to False, makes synchronous calls.
         """
         assert (
             self.communication_handle is None and not self.is_communication_outstanding
         ), 'Should not have multiple communication calls outstanding at once'
 
-        # Make sure norm of grads in bucket are not NaN
-        # prior to data-parallel all-reduce / reduce-scatter.
         if self.ddp_config.check_for_nan_in_grad:
-            global_rank = torch.distributed.get_rank()
-            norm = self.grad_data.norm(p=2)
-            assert not norm.isnan(), (
-                f'Rank {global_rank}: found NaN in local grad norm in '
-                f'backward pass before data-parallel communication collective. '
-                f'Device: {torch.cuda.current_device()}, node: {os.uname()[1]}'
-            )
+            self.check_for_nan_in_grad()
 
         # gradient_scaling_factor already takes into account whether we are computing
         # an average or sum in the data-parallel collective.
-        if self.gradient_scaling_factor != 1.0:
-            self.grad_data *= self.gradient_scaling_factor
+        for bucket in self.buckets:
+            if bucket.gradient_scaling_factor != 1.0:
+                bucket.grad_data *= bucket.gradient_scaling_factor
 
         # Decide reduce_op.
         reduce_op = torch.distributed.ReduceOp.SUM
         if self.ddp_config.average_in_collective:
             reduce_op = torch.distributed.ReduceOp.AVG
 
-        # Use async_op only when overlap_grad_reduce is True.
-        if self.ddp_config.use_distributed_optimizer:
-            local_data_view = shard_buffer(self.grad_data, self.data_parallel_world_size)[
-                self.data_parallel_rank
-            ]
-            self.communication_handle = torch.distributed._reduce_scatter_base(
-                local_data_view,
-                self.grad_data,
-                op=reduce_op,
-                group=self.data_parallel_group,
-                async_op=self.ddp_config.overlap_grad_reduce,
-            )
+        # Decide async_op
+        # Use async communications only when overlap_grad_reduce is True.
+        async_op = self.ddp_config.overlap_grad_reduce
+
+        with _coalescing_manager(self.data_parallel_group, async_ops=async_op) as cm:
+            for bucket in self.buckets:
+                if self.ddp_config.use_distributed_optimizer:
+                    local_data_view = shard_buffer(bucket.grad_data, self.data_parallel_world_size)[
+                        self.data_parallel_rank
+                    ]
+                    torch.distributed._reduce_scatter_base(
+                        local_data_view,
+                        bucket.grad_data,
+                        op=reduce_op,
+                        group=self.data_parallel_group,
+                        async_op=async_op,
+                    )
+                else:
+                    torch.distributed.all_reduce(
+                        bucket.grad_data,
+                        op=reduce_op,
+                        group=self.data_parallel_group,
+                        async_op=async_op,
+                    )
+        if async_op:
+            self.communication_handle = cm
         else:
-            self.communication_handle = torch.distributed.all_reduce(
-                self.grad_data,
-                op=reduce_op,
-                group=self.data_parallel_group,
-                async_op=self.ddp_config.overlap_grad_reduce,
-            )
+            # When using `_coalescing_manager`, even if a synchronous op (async_op=False) is used,
+            # `cm` is not None, which is different from when `_coalescing_manager` is not used in
+            # which case the torch.distributed._reduce_scatter_base() will return None. In order to
+            # maintain consistency with prior code, we need to manually set communication handle to
+            # None.
+            self.communication_handle = None
+
         if self.ddp_config.overlap_grad_reduce:
             self.is_communication_outstanding = True
         else:
@@ -157,13 +204,13 @@ def start_grad_sync(self):
 
     def finish_grad_sync(self):
         """
-        Finishes grad sync (all-reduce or reduce-scatter) communication operation
-        for this bucket.
+        Finishes grad sync (all-reduce or reduce-scatter) communication operations
+        for all buckets in the bucket group.
 
         When overlap_grad_reduce is set to True, waits for asynchronous communication
-        call to complete. When overlap_grad_reduce is set to False, makes synchronous call.
+        calls to complete. When overlap_grad_reduce is set to False, calls synchronous
+        communication ops.
         """
-        # If overlap_grad_reduce is False, start (and finish) synchronous communication call here.
         if not self.ddp_config.overlap_grad_reduce:
             self.start_grad_sync()
             return
@@ -180,15 +227,16 @@ def register_grad_ready(self, param: torch.nn.Parameter):
         When the number of microbatches is greater than 1, we only want to register
         grads as ready when processing the last microbatch and overlap_grad_reduce is True.
         """
-        assert param in self.params, 'Param is not in the bucket'
-        assert param not in self.params_with_grad, 'Cannot set grad twice'
         assert (
             self.ddp_config.overlap_grad_reduce
-        ), 'register_grad_ready() should be called only when overlapping grad reduce'
-        self.params_with_grad.add(param)
-        # If all params in bucket have grads available, issue communication call.
-        if len(self.params_with_grad) == len(self.params):
-            self.start_grad_sync()
+        ), 'register_grad_ready() should only be called when overlap_grad_reduce is True'
+        if self.is_last_microbatch:
+            assert param in self.param_to_bucket, 'Param is not in the bucket group'
+            assert param not in self.params_with_grad, 'Cannot set grad twice'
+            self.params_with_grad.add(param)
+            # If all params in bucket group have grads available, issue communication call.
+            if len(self.params_with_grad) == len(self.params):
+                self.start_grad_sync()
 
 
 class ParamAndGradBuffer:
@@ -208,6 +256,9 @@ class ParamAndGradBuffer:
         gradient_scaling_factor: This factor is utilized to scale gradients prior to their
             communication. Its application is twofold: it facilitates the averaging of gradients
             and the scaling of gradients in the context of the Mixture of Experts (MoE) model.
+        param_indices: The index of each param among the params with same dtype, if a param is fp8,
+            use its "fake" high precision dtype to determine which params have same dtype with it.
+            These indices are needed when loading a non-native-fp8 checkpoint in native-fp8 mode.
     """
 
     def __init__(
@@ -220,8 +271,11 @@ def __init__(
         bucket_size: int,
         param_to_name: Dict[torch.nn.Parameter, str],
         gradient_scaling_factor: float,
+        param_indices: List[int],
     ):
         self.ddp_config = ddp_config
+        self.params = params
+        self.param_indices = param_indices
 
         # Check that params are unique.
         unique_params = set()
@@ -238,7 +292,6 @@ def __init__(
             group=self.data_parallel_group
         )
         self.gradient_scaling_factor = gradient_scaling_factor
-        self.is_last_microbatch = True
 
         # Data structures to store underlying buckets and relevant indexing data.
         self.buckets = []
@@ -374,7 +427,7 @@ def _does_param_require_new_bucket(param):
         )
 
         # Finally, map param.data and param.main_grad fields to buffers.
-        bucket_params = set()
+        bucket_params = []
         bucket_data_start_index = 0
         cur_bucket_id = 0
         for param in params[::-1]:
@@ -385,9 +438,13 @@ def _does_param_require_new_bucket(param):
             # Assign param.data to appropriate segment of self.param_data.
             if self.param_data is not None:
                 old_param_data = param.data
-                param.data = self._get(
+                new_param_data = self._get(
                     param.data.shape, data_start_index, buffer_type=BufferType.PARAM
                 )
+                if is_float8tensor(param):
+                    param._data = new_param_data
+                else:
+                    param.data = new_param_data
                 assert old_param_data._base is None
                 # Copy tensor values (from initialization or checkpoint).
                 param.data.detach().copy_(old_param_data)
@@ -406,11 +463,11 @@ def _does_param_require_new_bucket(param):
                     bucket_id=cur_bucket_id,
                 )
                 bucket_data_start_index = bucket_data_end_index
-                bucket_params = set()
+                bucket_params = []
                 assert cur_bucket_id + 1 == len(self.buckets)
                 assert bucket_id == cur_bucket_id + 1
                 cur_bucket_id = bucket_id
-            bucket_params.add(param)
+            bucket_params.append(param)
 
         # Add remaining params to a new bucket.
         if len(bucket_params) > 0:
@@ -488,14 +545,11 @@ def _set_bucket(
             torch.Size([end_index - start_index]), start_index, buffer_type=BufferType.GRAD
         )
         bucket = Bucket(
-            ddp_config=self.ddp_config,
             params=bucket_params,
             param_data=bucketed_param_data,
             grad_data=bucketed_grad_data,
             offset=start_index,
             numel_unpadded=numel_unpadded,
-            data_parallel_group=self.data_parallel_group,
-            data_parallel_world_size=self.data_parallel_world_size,
             gradient_scaling_factor=self.gradient_scaling_factor,
         )
         self.buckets.append(bucket)
@@ -505,48 +559,84 @@ def _set_bucket(
 
     def reset(self):
         """
-        Zero out the underlying grad_buffer and reset all buckets in preparation for the next
-        iteration of training.
+        Zero out the underlying grad_buffer.
         """
         self.grad_data.zero_()
-        for bucket in self.buckets:
-            bucket.reset()
-        self.is_last_microbatch = True
-
-    def start_grad_sync(self):
-        """
-        Initiates grad sync (all-reduce or reduce-scatter) communication operations
-        for all buckets in the grad buffer.
 
-        When overlap_grad_reduce is set to True, dispatches asynchronous communication
-        calls. When overlap_grad_reduce is set to False, calls synchronous
-        communication ops.
-        """
-        for bucket in self.buckets:
-            bucket.start_grad_sync()
 
-    def finish_grad_sync(self):
-        """
-        Finishes grad sync (all-reduce or reduce-scatter) communication operations
-        for all buckets in the grad buffer.
-
-        When overlap_grad_reduce is set to True, waits for asynchronous communication
-        calls to complete. When overlap_grad_reduce is set to False, calls synchronous
-        communication ops.
-        """
-        for bucket in self.buckets:
-            bucket.finish_grad_sync()
-
-    def register_grad_ready(self, param: torch.nn.Parameter):
-        """
-        Registers grads for the passed-in param to be "ready" for grad sync.
+def partition_buckets(buffers: List[ParamAndGradBuffer]) -> List[BucketGroup]:
+    """
+    Automatically regroups the buckets of input buffers and returns a list of `BucketGroup`.
+
+    In some scenarios, we need to put buckets from different buffers into a group so that their
+    communication can be aggregated.
+
+    For example, when there are both fp8 weights and bf16 biases in the model and vpp is enabled,
+    each model chunk will have an fp8 bucket and a bf16 bucket, which doubles the number of
+    communication kernels, and because of the use of CUDA_DEVICE_MAX_CONNECTIONS=1, having multiple
+    back-to-back communications will prevent the overlap of the communication kernels with
+    computation kernels.
+
+    The grouping strategy is:
+    1. When there is no fp8 buffer in the input buffers, let each BucketGroup have only one
+       bucket.
+    2. When using fp8 params, merge all non-fp8 buckets into the last fp8 bucket group.
+       - Since the non-fp8 parameters (typically the biases of various layers) are relatively
+         small, they are likely to be grouped into a single non-fp8 bucket.
+       - The fp8 buckets start from the end of the model, i.e., the first bucket corresponds to
+         the end of the model, while the last bucket corresponds to the beginning.
+       - If we combine the non-fp8 bucket with the first fp8 bucket, we cannot initiate the
+         reduce-scatter to synchronize gradients after the backward pass at the end of the model
+         has completed. This is because we need to wait for the non-fp8 params from the beginning
+         layers to obtain their gradients.
+       - Combining the non-fp8 bucket with the last fp8 bucket can help avoid this issue.
+    """
 
-        When the number of microbatches is greater than 1, we only want to register
-        grads as ready when processing the last microbatch and overlap_grad_reduce is True.
-        """
-        assert (
-            self.ddp_config.overlap_grad_reduce
-        ), 'register_grad_ready() should only be called when overlap_grad_reduce is True'
-        if self.is_last_microbatch:
-            bucket = self.param_to_bucket[param]
-            bucket.register_grad_ready(param)
+    dtype_to_buffer_map = {}
+    for buffer in buffers:
+        dtype = buffer.param_dtype
+        # Make sure that the param_dtype of any two buffers is different.
+        assert dtype not in dtype_to_buffer_map
+        dtype_to_buffer_map[dtype] = buffer
+
+    if torch.uint8 not in dtype_to_buffer_map:
+        # Case 1: When there is no fp8 buffer in the input buffers, let each BucketGroup have only
+        #         one bucket.
+        bucket_groups = []
+        for buffer in buffers:
+            for bucket in buffer.buckets:
+                bucket_groups.append(
+                    BucketGroup(
+                        [bucket],
+                        buffer.ddp_config,
+                        buffer.data_parallel_group,
+                        buffer.data_parallel_world_size,
+                    )
+                )
+        return bucket_groups
+    else:
+        # Case 2: When using fp8 params, merge all non-fp8 buckets into the last fp8 bucket group.
+        non_fp8_buckets = []
+        for buffer in buffers:
+            if buffer.param_dtype != torch.uint8:
+                for bucket in buffer.buckets:
+                    non_fp8_buckets.append(bucket)
+
+        bucket_groups = []
+        fp8_buffer = dtype_to_buffer_map[torch.uint8]
+        for bucket in fp8_buffer.buckets:
+            if len(bucket_groups) == len(fp8_buffer.buckets) - 1:
+                # The last bucket group.
+                group_buckets = [bucket] + non_fp8_buckets
+            else:
+                # The first N-1 bucket groups.
+                group_buckets = [bucket]
+            bucket_groups.append(
+                BucketGroup(
+                    group_buckets,
+                    buffer.ddp_config,
+                    buffer.data_parallel_group,
+                    buffer.data_parallel_world_size,
+                )
+            )
+        return bucket_groups
diff --git a/megatron/core/optimizer/__init__.py b/megatron/core/optimizer/__init__.py
index d06911f1b9..6de51def31 100644
--- a/megatron/core/optimizer/__init__.py
+++ b/megatron/core/optimizer/__init__.py
@@ -19,6 +19,7 @@
         )
 
         ## apex's FusedAdam is a drop-in replacement for torch's AdamW
+        # pylint: disable-next=line-too-long
         ## see https://github.com/NVIDIA/apex/blob/7b73b12361068a10b0f44844534613f252a5ea75/apex/optimizers/fused_adam.py#L16
         from torch.optim import AdamW as Adam, SGD
 
@@ -107,7 +108,8 @@ def _get_param_groups(
                 wd_mult, _lr_mult = 0.0, lr_mult
 
             is_decoupled_lr = False
-            # For input/embedding and output layer: embedding.word_embeddings.weight / output_layer.weight.
+            # For input/embedding and output layer: embedding.word_embeddings.weight /
+            # output_layer.weight.
             if use_decoupled_learning_rate and getattr(
                 param, 'is_embedding_or_output_parameter', False
             ):
@@ -189,7 +191,7 @@ def _get_param_groups_and_buffers(
     lr_mult: float,
     filter_fn: Callable,
     buffer_name: str,
-) -> Tuple[List[Dict], Dict[int, ParamAndGradBuffer]]:
+) -> Tuple[List[Dict], Dict[int, List[ParamAndGradBuffer]]]:
     """Returns parameter groups and buffer for optimizer.
 
     Args:
diff --git a/megatron/core/optimizer/distrib_optimizer.py b/megatron/core/optimizer/distrib_optimizer.py
index c211619d0e..a51b15e4f3 100644
--- a/megatron/core/optimizer/distrib_optimizer.py
+++ b/megatron/core/optimizer/distrib_optimizer.py
@@ -9,6 +9,7 @@
 from typing import Callable, Dict, List, Optional, Tuple
 
 import torch
+from torch.distributed import _coalescing_manager
 
 HAVE_APEX_OR_TE = True
 try:
@@ -31,13 +32,25 @@
     ShardedStateDict,
     ShardedTensorFactory,
 )
-from ..dist_checkpointing.optimizer import get_param_id_to_sharded_param_map
 from ..dist_checkpointing.utils import extract_sharded_tensors_and_factories
-from ..distributed import ParamAndGradBuffer, shard_buffer
+from ..distributed import ParamAndGradBuffer, partition_buckets, shard_buffer
+from ..utils import is_float8tensor
 from .grad_scaler import MegatronGradScaler
-from .optimizer import MixedPrecisionOptimizer, _zero_grad_group_helper
+from .optimizer import (
+    MixedPrecisionOptimizer,
+    _multi_tensor_copy_this_to_that,
+    _zero_grad_group_helper,
+)
 from .optimizer_config import OptimizerConfig
 
+try:
+    # This will be used when "--fp8-param-gather" is enabled.
+    # When BF16/FP16 parameters don't exist, we need to cast the FP32 main parameters to
+    # FP8 directly in the optimizer.
+    from transformer_engine.pytorch.cpp_extensions import cast_to_fp8
+except:
+    pass
+
 logger = getLogger(__name__)
 
 
@@ -220,9 +233,10 @@ def _build_model_param_gbuf_map(
             for dtype, gbuf_range_map_for_all_buckets in gbuf_range_map.items():
                 for bucket_index, gbuf_range_map in enumerate(gbuf_range_map_for_all_buckets):
                     for param, _ in gbuf_range_map["param_map"].items():
-                        assert (
-                            param not in param_gbuf_map
-                        ), "Param should not be in param_gbuf_map; each param only belongs to a single bucket"
+                        assert param not in param_gbuf_map, (
+                            "Param should not be in param_gbuf_map; "
+                            "each param only belongs to a single bucket"
+                        )
                         param_gbuf_map[param] = (gbuf_index, dtype, bucket_index)
         return param_gbuf_map
 
@@ -333,7 +347,25 @@ def _build_model_and_main_param_groups(
                     shard_model_param = model_param.detach().view(-1)[
                         param_range.start : param_range.end
                     ]
-                    shard_main_param = shard_model_param.clone().float()
+
+                    # If we use FP8 params to initialize FP32 main params (compared to using the
+                    # bf16/fp16 params to initialize the main params), there will be a loss of
+                    # precision at the beginning of training (this problem will not occur if the
+                    # training is long enough or if the main params are loaded from a checkpoint).
+                    if is_float8tensor(model_param) and hasattr(
+                        model_param, 'get_high_precision_init_val'
+                    ):
+                        shard_main_param = (
+                            model_param.get_high_precision_init_val()
+                            .view(-1)[param_range.start : param_range.end]
+                            .clone()
+                            .to(shard_model_param.device)
+                            .float()
+                        )
+                        model_param.clear_high_precision_init_val()
+                    else:
+                        shard_main_param = shard_model_param.clone().float()
+
                     tensor_parallel.copy_tensor_model_parallel_attributes(
                         shard_model_param, model_param
                     )
@@ -447,12 +479,18 @@ def __init__(
         self.data_parallel_group = data_parallel_group
         self.data_parallel_group_gloo = data_parallel_group_gloo
         self.data_parallel_group_idx = data_parallel_group_idx
+
         self.gbuf_idx_to_model_idx_map = {}
         gbuf_idx = 0
         for model_idx, buffers in self.per_model_buffers.items():
             for _ in buffers:
                 self.gbuf_idx_to_model_idx_map[gbuf_idx] = model_idx
                 gbuf_idx += 1
+
+        self.per_model_bucket_groups = {}
+        for model_idx, buffers in self.per_model_buffers.items():
+            self.per_model_bucket_groups[model_idx] = partition_buckets(buffers)
+
         self.gbuf_ranges = []
         self.per_bucket_numel = []
         self.per_bucket_numel_unpadded = []
@@ -499,23 +537,23 @@ def __init__(
         self.param_to_all_gather_handle_index_map = {}
 
         self.pbuf_view_items = self._get_model_param_buffer_dp_views()
-        for gbuf_index, dtype, bucket_index, _, _ in self.pbuf_view_items:
+        for model_idx, dtypes, bucket_group_index, _, _ in self.pbuf_view_items:
             self.all_gather_handle_index_to_bucket_index_map.append(
-                (gbuf_index, dtype, bucket_index)
+                (model_idx, dtypes, bucket_group_index)
             )
             all_gather_handle_index = len(self.all_gather_handle_index_to_bucket_index_map) - 1
             self.all_gather_handles.append(None)
 
             # Store all all_gather_handle_indices.
-            model_idx = self.gbuf_idx_to_model_idx_map[gbuf_index]
             if model_idx not in self.model_index_to_all_gather_handle_index_map:
                 self.model_index_to_all_gather_handle_index_map[model_idx] = []
             self.model_index_to_all_gather_handle_index_map[model_idx].append(
                 all_gather_handle_index
             )
 
-            for param in self.buffers[gbuf_index].buckets[bucket_index].params_list:
-                self.param_to_all_gather_handle_index_map[param] = all_gather_handle_index
+            for bucket in self.per_model_bucket_groups[model_idx][bucket_group_index].buckets:
+                for param in bucket.params_list:
+                    self.param_to_all_gather_handle_index_map[param] = all_gather_handle_index
         self.num_all_gather_handles = len(self.all_gather_handle_index_to_bucket_index_map)
 
         self.overlap_param_gather = self.config.overlap_param_gather
@@ -865,9 +903,9 @@ def get_parameter_state_dp_zero(self):
                         # Concatenate.
                         if data_parallel_rank == 0:
                             recv_tensors_concatenated = torch.cat(recv_tensors)
-                            # Copy this bucket's collected all-gather tensors into the right place in the
-                            # tensor for the buffer. The tensor for the buffer gets rid of the padding
-                            # between buckets.
+                            # Copy this bucket's collected all-gather tensors into the right place
+                            # in the tensor for the buffer. The tensor for the buffer gets rid of
+                            # the padding between buckets.
                             start = offset_in_world_tensors
                             end = offset_in_world_tensors + gbuf_world_numel_unpadded
                             world_tensors[key][start:end].copy_(
@@ -993,7 +1031,7 @@ def sharded_param_state_fs_bucket_space(
         # per_bucket_numel metadata is saved separately for each TPxPP domain.
         for per_bucket_key in ('per_bucket_numel', 'per_bucket_numel_unpadded'):
             state[per_bucket_key] = ShardedObject(
-                f'optimizer.distributed.dp_group_idx_{self.data_parallel_group_idx}.{per_bucket_key}',
+                f'optimizer.distributed.dp_group_idx_{self.data_parallel_group_idx}.{per_bucket_key}',  # pylint: disable=line-too-long
                 state[per_bucket_key],
                 (1,),
                 (0,),
@@ -1008,7 +1046,7 @@ def sharded_param_state_fs_bucket_space(
                     assert gbuf_world_numel % data_parallel_world_size == 0
                     gbuf_local_numel = gbuf_world_numel // data_parallel_world_size
 
-                    sharded_bucket_key = f'optimizer.distributed.dp_group_idx_{self.data_parallel_group_idx}.gbuf_idx_{gbuf_idx}.dtype_{dtype}.bucket_idx_{bucket_idx}'
+                    sharded_bucket_key = f'optimizer.distributed.dp_group_idx_{self.data_parallel_group_idx}.gbuf_idx_{gbuf_idx}.dtype_{dtype}.bucket_idx_{bucket_idx}'  # pylint: disable=line-too-long
 
                     # The global ckpt tensors must be fully covered.
                     # We add extra empty padding if necessary
@@ -1109,7 +1147,9 @@ def sharded_param_state_fs_model_space(
 
         prefix = 'optimizer.state'
         state = {}
-        param_idx = 0  # this is not stored in the checkpoint, used only to identify params in `sharded_param_state_fs_model_space`
+        # this is not stored in the checkpoint, used only to identify params in
+        # `sharded_param_state_fs_model_space`
+        param_idx = 0
         for gbuf_range_maps in self.gbuf_ranges:
             for gbuf_range_map_for_all_buckets in gbuf_range_maps.values():
                 for gbuf_range_map in gbuf_range_map_for_all_buckets:
@@ -1121,7 +1161,8 @@ def sharded_param_state_fs_model_space(
                         optim_state = self.optimizer.state[main_param]
 
                         tensors = {"fp32_param": main_param, **optim_state}
-                        # Match optimizer parameter with model ShardedTensor (or ShardedTensorFactory)
+                        # Match optimizer parameter with model ShardedTensor (or
+                        # ShardedTensorFactory)
                         try:
                             sharded_metadata = param_to_sharded_metadata[model_param]
                         except KeyError as e:
@@ -1240,7 +1281,8 @@ def _update_legacy_world_tensors(cls, old_tensors, new_numels):
         return new_tensors
 
     def load_parameter_state_from_dp_zero_legacy(self, state_dict):
-        """Load parameter state (i.e., parameter & optimizer tensors) from DP 0 rank, using the legacy checkpoint format as described below.
+        """Load parameter state (i.e., parameter & optimizer tensors) from DP 0 rank, using the
+        legacy checkpoint format as described below.
 
         The difference between this method and `load_parameter_state_from_dp_zero_modern()`
         is that this method is used for updating the format of checkpoints that
@@ -1309,7 +1351,8 @@ def load_parameter_state_from_dp_zero_legacy(self, state_dict):
                             ), "%d vs. %d." % (world_tensor.numel(), gbuf_world_numel_unpadded)
                             offset_in_world_tensors += gbuf_world_numel_unpadded
 
-                            # Pad world_tensor to gbuf_world_numel. Don't pad at the front, pad at the back.
+                            # Pad world_tensor to gbuf_world_numel. Don't pad at the front, pad at
+                            # the back.
                             world_tensor = torch.nn.functional.pad(
                                 world_tensor, (0, gbuf_world_numel - gbuf_world_numel_unpadded)
                             )
@@ -1375,6 +1418,10 @@ def load_parameter_state_from_dp_zero(self, state_dict, *, update_legacy_format=
             self.data_parallel_group_gloo
         )
 
+        if data_parallel_rank == 0:
+            # Do nothing if "--fp8-param-gather" is not used.
+            self.split_state_dict_if_needed(state_dict)
+
         # Scatter tensors to all DP ranks.
         for gbuf_idx, gbuf_range_maps in enumerate(self.gbuf_ranges):
             for dtype, gbuf_range_map_for_all_buckets in gbuf_range_maps.items():
@@ -1414,7 +1461,8 @@ def load_parameter_state_from_dp_zero(self, state_dict, *, update_legacy_format=
                             world_tensor = world_tensors[start:end]
                             offset_in_world_tensors += gbuf_world_numel_unpadded
 
-                            # Pad world_tensor to gbuf_world_numel. Don't pad at the front, pad at the back.
+                            # Pad world_tensor to gbuf_world_numel. Don't pad at the front, pad at
+                            # the back.
                             world_tensor = torch.nn.functional.pad(
                                 world_tensor, (0, gbuf_world_numel - gbuf_world_numel_unpadded)
                             )
@@ -1455,6 +1503,139 @@ def load_parameter_state_from_dp_zero(self, state_dict, *, update_legacy_format=
                                 recv_tensor[gbuf_local_start:gbuf_local_end]
                             )
 
+    def split_state_dict_if_needed(self, state_dict):
+        """
+        When "--fp8-param-gather" is disabled, weights and biases are stored in the same
+        `ParamAndGradBuffer`. So, when saving a checkpoint, the optimizer's main parameters are
+        saved in a single continuous tensor (this also applies to "exp_avg" and "exp_avg_sq").
+
+        However, when "--fp8-param-gather" is enabled, weights(in fp8 dtype) and biases(in bf16/fp16
+        dtype) are stored in separate `ParamAndGradBuffer`. Therefore, when we enabled
+        "--fp8-param-gather", and want to load a checkpoint saved without "--fp8-param-gather", we
+        need to split the weights(fp8) and biases(bf16/fp16) in the static_dict into two separate
+        tensors.
+        """
+        # Skip if there is no fp8 buffers.
+        fp8_gbuf_indices = []
+        for gbuf_idx, gbuf_range_maps in enumerate(self.gbuf_ranges):
+            for dtype, _ in gbuf_range_maps.items():
+                if is_float8tensor(self.buffers[gbuf_idx].params[0]):
+                    fp8_gbuf_indices.append(gbuf_idx)
+        if len(fp8_gbuf_indices) == 0:
+            return
+
+        dtype_to_gbuf_idx = {}
+        for key in state_dict.keys():
+            if key != 'buckets_coalesced':
+                for dtype in state_dict[key].keys():
+                    assert dtype not in dtype_to_gbuf_idx
+                    if dtype[0] == torch.uint8:
+                        # If the `state_dict`` already contains a torch.uint8 buffer, we assumed
+                        # that the fp8 weights and fp16/bf16 biases in the checkpoint are already
+                        # separated. In this case, no action is required, so we can return directly.
+                        return
+                    dtype_to_gbuf_idx[dtype] = key
+
+        # 1. Replace the gbuf_idx in the checkpoint with the new gbuf_idx.
+        # 2. Copy the non-tensor data (i.e., the "buckets_coalesced") to `new_state_dict`.
+        new_state_dict = {'buckets_coalesced': state_dict['buckets_coalesced']}
+        for gbuf_idx, gbuf_range_maps in enumerate(self.gbuf_ranges):
+            for dtype, _ in gbuf_range_maps.items():
+                if not is_float8tensor(self.buffers[gbuf_idx].params[0]):
+                    new_state_dict[gbuf_idx] = state_dict[dtype_to_gbuf_idx[dtype]]
+
+        for fp8_gbuf_idx in fp8_gbuf_indices:
+            # Note that `self.buffers[fp8_gbuf_idx].params[0].dtype` is the dummy dtype of
+            # `Float8Tensor`, not torch.uint8.
+            non_fp8_param_and_grad_dtype = (
+                self.buffers[fp8_gbuf_idx].params[0].dtype,
+                self.buffers[fp8_gbuf_idx].grad_dtype,
+            )
+
+            # Iterate through all buffers to find the one that needs to be split.
+            non_fp8_gbuf_idx = None
+            for gbuf_idx, gbuf_range_maps in enumerate(self.gbuf_ranges):
+                for dtype, _ in gbuf_range_maps.items():
+                    if dtype == non_fp8_param_and_grad_dtype:
+                        non_fp8_gbuf_idx = gbuf_idx
+            assert non_fp8_gbuf_idx is not None
+
+            # We need the fp8_flags to determine the order of weight (fp8) and bias (fp16/bf16) in
+            # the buffer.
+            index_to_fp8_map = {}
+            for index in self.buffers[fp8_gbuf_idx].param_indices:
+                assert index not in index_to_fp8_map
+                index_to_fp8_map[index] = True
+            for index in self.buffers[non_fp8_gbuf_idx].param_indices:
+                assert index not in index_to_fp8_map
+                index_to_fp8_map[index] = False
+            param_indices = (
+                self.buffers[fp8_gbuf_idx].param_indices
+                + self.buffers[non_fp8_gbuf_idx].param_indices
+            )
+            assert min(param_indices) == 0
+            assert max(param_indices) == len(param_indices) - 1
+            fp8_flags = []
+            for i in range(len(param_indices)):
+                fp8_flag.append(index_to_fp8_map[i])
+
+            fp8_buffer = self.buffers[fp8_gbuf_idx]
+            non_fp8_buffer = self.buffers[non_fp8_gbuf_idx]
+
+            fp8_idx = len(fp8_buffer.params) - 1
+            non_fp8_idx = len(non_fp8_buffer.params) - 1
+            offsets, fp8_offsets, non_fp8_offsets = [0], [0], [0]
+
+            # Because the parameters in `ParamAndGradBuffer` are traversed in reverse order, the
+            # flag here also needs to be traversed in reverse order.
+            for fp8_flag in fp8_flags[::-1]:
+                if fp8_flag:
+                    numel = fp8_buffer.params[fp8_idx].nelement()
+                    fp8_idx -= 1
+                    offsets.append(offsets[-1] + numel)
+                    fp8_offsets.append(fp8_offsets[-1] + numel)
+                else:
+                    numel = non_fp8_buffer.params[non_fp8_idx].nelement()
+                    non_fp8_idx -= 1
+                    offsets.append(offsets[-1] + numel)
+                    non_fp8_offsets.append(non_fp8_offsets[-1] + numel)
+
+            # Split the target buffer into two separate buffers.
+            fp8_state_dict, non_fp8_state_dict = {}, {}
+            for key in ['param', 'exp_avg', 'exp_avg_sq']:
+                tensor = state_dict[non_fp8_gbuf_idx][non_fp8_param_and_grad_dtype][key]
+                fp8_tensor = torch.empty([fp8_offsets[-1]], dtype=tensor.dtype)
+                non_fp8_tensor = torch.empty([non_fp8_offsets[-1]], dtype=tensor.dtype)
+
+                fp8_idx, non_fp8_idx = 0, 0
+                for i in range(len(offsets) - 1):
+                    if fp8_flags[-(i + 1)]:
+                        fp8_tensor[fp8_offsets[fp8_idx] : fp8_offsets[fp8_idx + 1]].copy_(
+                            tensor[offsets[i] : offsets[i + 1]]
+                        )
+                        fp8_idx += 1
+                    else:
+                        non_fp8_tensor[
+                            non_fp8_offsets[non_fp8_idx] : non_fp8_offsets[non_fp8_idx + 1]
+                        ].copy_(tensor[offsets[i] : offsets[i + 1]])
+                        non_fp8_idx += 1
+
+                fp8_state_dict[key] = fp8_tensor
+                non_fp8_state_dict[key] = non_fp8_tensor
+
+            fp8_state_dict['numel_unpadded'] = fp8_offsets[-1]
+            non_fp8_state_dict['numel_unpadded'] = non_fp8_offsets[-1]
+
+            # Add the two separate buffers into `new_state_dict`.
+            new_state_dict[fp8_gbuf_idx] = {}
+            new_state_dict[fp8_gbuf_idx][(torch.uint8, fp8_buffer.grad_dtype)] = fp8_state_dict
+            new_state_dict[non_fp8_gbuf_idx][non_fp8_param_and_grad_dtype] = non_fp8_state_dict
+
+        # Inplace update state_dict
+        state_dict.clear()
+        for key, value in new_state_dict.items():
+            state_dict[key] = value
+
     def load_parameter_state(self, filename: str, *, update_legacy_format=False):
         """Load the distributed parameter state from disk.
 
@@ -1522,29 +1703,42 @@ def _get_model_param_buffer_dp_views(self):
         """
 
         # Buffer views.
-        # Add in reverse order in each model chunk since buckets start from the end of the model but we want
-        # all-gathers to run first for the start of the model (same order as forward pass).
-        # We keep the view_items in model chunk order since we want to still first run all_gather and
-        # all_gather_handle.wait() for the first model chunk.
-        # In all cases, we want all_gather and all_gather_handle.wait() to be called in the same order,
-        # and all_gather_handle.wait() needs to be called just before the corresponding forward pass.
+        # Add in reverse order in each model chunk since buckets start from the end of the model
+        # but we want all-gathers to run first for the start of the model (same order as forward
+        # pass).
+        # We keep the view_items in model chunk order since we want to still first run all_gather
+        # and all_gather_handle.wait() for the first model chunk.
+        # In all cases, we want all_gather and all_gather_handle.wait() to be called in the same
+        # order, and all_gather_handle.wait() needs to be called just before the corresponding
+        # forward pass.
         view_items = []
-        for gbuf_index, buffer in enumerate(self.buffers):
+        for model_idx, bucket_groups in self.per_model_bucket_groups.items():
             view_items_per_model_chunk = []
-            dtype = self.buffers[gbuf_index].param_dtype
-            for bucket_index, bucket in enumerate(buffer.buckets):
-                data_parallel_world_size = torch.distributed.get_world_size(
-                    self.data_parallel_group
-                )
-                buf_views = shard_buffer(bucket.param_data, data_parallel_world_size)
+            for bucket_group_idx, bucket_group in enumerate(bucket_groups):
+                dtypes = []
+                bucket_data = []
+                buf_views = []
+                for bucket in bucket_group.buckets:
+                    dtypes.append(bucket.param_data.dtype)
+                    data_parallel_world_size = torch.distributed.get_world_size(
+                        self.data_parallel_group
+                    )
+                    buf_view = shard_buffer(bucket.param_data, data_parallel_world_size)
+                    bucket_data.append(bucket.param_data)
+                    buf_views.append(buf_view)
                 view_items_per_model_chunk.insert(
-                    0, (gbuf_index, dtype, bucket_index, bucket.param_data, buf_views)
+                    0, (model_idx, dtypes, bucket_group_idx, bucket_data, buf_views)
                 )
             view_items.extend(view_items_per_model_chunk)
 
         return view_items
 
-    def _dispatch_gather_model_params(self, all_gather_handle_index: int, force_sync: bool = False):
+    def _dispatch_gather_model_params(
+        self,
+        all_gather_handle_index: int,
+        force_sync: bool = False,
+        already_in_coalescing_manager: bool = False,
+    ):
         """
         All-gather updated model params.
 
@@ -1562,18 +1756,40 @@ def _dispatch_gather_model_params(self, all_gather_handle_index: int, force_sync
             # across all data-parallel ranks, due to padding done in
             # param_and_grad_buffer.py). Thus, all sub-views will have consistent
             # start / end indexes across data-parallel ranks.
-            (gbuf_index, dtype, bucket_index, pbuf, pbuf_views) = self.pbuf_view_items[
-                all_gather_handle_index
-            ]
-            assert all_gather_handle_index < len(self.all_gather_handles)
-            all_gather_handle = torch.distributed._all_gather_base(
-                pbuf, pbuf_views[data_parallel_rank], group=data_parallel_group, async_op=async_op
+            (model_index, dtypes, bucket_group_index, pbuf_list, pbuf_views_list) = (
+                self.pbuf_view_items[all_gather_handle_index]
             )
-            self.all_gather_handles[all_gather_handle_index] = all_gather_handle
+            assert all_gather_handle_index < len(self.all_gather_handles)
+            if not already_in_coalescing_manager:
+                with _coalescing_manager(data_parallel_group, async_ops=async_op) as cm:
+                    for i in range(len(pbuf_list)):
+                        torch.distributed._all_gather_base(
+                            pbuf_list[i],
+                            pbuf_views_list[i][data_parallel_rank],
+                            group=data_parallel_group,
+                            async_op=async_op,
+                        )
+                if async_op:
+                    self.all_gather_handles[all_gather_handle_index] = cm
+                else:
+                    # When using `_coalescing_manager`, even if a synchronous op (async_op=False)
+                    # is used, `cm` is not None, which is different from when `_coalescing_manager`
+                    # is not used in which case the torch.distributed._reduce_scatter_base() will
+                    # return None. In order to maintain consistency with prior code, we need to
+                    # manually set communication handel to None.
+                    self.all_gather_handles[all_gather_handle_index] = None
+            else:
+                for i in range(len(pbuf_list)):
+                    torch.distributed._all_gather_base(
+                        pbuf_list[i],
+                        pbuf_views_list[i][data_parallel_rank],
+                        group=data_parallel_group,
+                        async_op=async_op,
+                    )
             assert self.all_gather_handle_index_to_bucket_index_map[all_gather_handle_index] == (
-                gbuf_index,
-                dtype,
-                bucket_index,
+                model_index,
+                dtypes,
+                bucket_group_index,
             )
 
     def _make_forward_pre_hook(self):
@@ -1634,7 +1850,9 @@ def start_param_sync(self, model_index: int, *unused, force_dispatch: bool = Fal
                 group=self.data_parallel_group, async_ops=self.overlap_param_gather
             ) as cm:
                 for all_gather_handle_index in all_gather_handle_indices:
-                    self._dispatch_gather_model_params(all_gather_handle_index)
+                    self._dispatch_gather_model_params(
+                        all_gather_handle_index, already_in_coalescing_manager=True
+                    )
             if self.overlap_param_gather:
                 for all_gather_handle_index in all_gather_handle_indices:
                     self.all_gather_handles[all_gather_handle_index] = cm
@@ -1737,7 +1955,26 @@ def copy_group_params(shard_main_groups, model_groups):
                         world_range.start : world_range.end
                     ]
 
-                    shard_model_param.data.copy_(shard_main_param)
+                    if is_float8tensor(model_param):
+                        # 1. When "--fp8-param-gather" is disabled, the main param is first cast to
+                        #    BF16/FP16, and then cast to FP8, so the amax_history is calculated
+                        #    using BF16/FP16 param.
+                        # 2. When "--fp8-param-gather" is enabled, we can cast the FP32 main param
+                        #    to FP8 directly, which results in slightly different results with
+                        #    higher speed. In theory, this does not affect convergence.
+                        # TODO: The following code maintains the logic of the point-1 above. It can
+                        # be deleted if it is not necessary.
+                        shard_main_param = shard_main_param.to(model_param.dtype)
+
+                        cast_to_fp8(
+                            shard_main_param.view(1, -1),
+                            model_param._fp8_meta['scaling_fwd'],
+                            model_param._fp8_meta_index,
+                            model_param._fp8_dtype,
+                            out=shard_model_param.view(1, -1),
+                        )
+                    else:
+                        shard_model_param.data.copy_(shard_main_param)
 
         # Copy shard groups to model groups.
         copy_group_params(self.shard_fp32_from_float16_groups, self.model_float16_groups)
@@ -1781,6 +2018,48 @@ def _reset_metadata_and_sync_gather_all_model_params(self, force_sync: bool):
             for all_gather_handle_index in range(len(self.all_gather_handles)):
                 self._dispatch_gather_model_params(all_gather_handle_index, force_sync=force_sync)
 
+    def _update_fp8_scale_inv_and_amax(self):
+        """
+        If detect FP8 parameters, update their `_scale_inv` and do reduce-max for their
+        `amax_history`.
+        """
+        amaxes = []
+        scales = []
+        scale_invs = []
+        # Iterate over all parameters inside this optimizer to find FP8 parameters.
+        for buffer in self.buffers:
+            for bucket in buffer.buckets:
+                for param in bucket.params_list:
+                    if is_float8tensor(param):
+                        fp8_meta = param._fp8_meta['scaling_fwd']
+                        fp8_meta_index = param._fp8_meta_index
+                        amaxes.append(fp8_meta.amax_history[0][fp8_meta_index].view(1))
+                        scales.append(fp8_meta.scale[fp8_meta_index].view(1))
+                        scale_invs.append(param._scale_inv.view(1))
+                        # Reset transpose cache
+                        param._reset_caches()
+
+        # If there is no FP8 parameters, skip all operations.
+        if len(scales) > 0:
+            dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device='cuda')
+
+            # Update scaling factors.
+            packed_scales = torch.empty(len(scales), dtype=torch.float32, device=scales[0].device)
+            packed_scale_views = [packed_scales[i].view(1) for i in range(len(scales))]
+            _multi_tensor_copy_this_to_that(scales, packed_scale_views, dummy_overflow_buf)
+            torch.reciprocal(packed_scales, out=packed_scales)
+            _multi_tensor_copy_this_to_that(packed_scale_views, scale_invs, dummy_overflow_buf)
+
+            # Reduce amaxes.
+            # Note: Assume each param has a separate amax.
+            packed_amaxes = torch.empty(len(amaxes), dtype=torch.float32, device=amaxes[0].device)
+            packed_amax_views = [packed_amaxes[i].view(1) for i in range(len(amaxes))]
+            _multi_tensor_copy_this_to_that(amaxes, packed_amax_views, dummy_overflow_buf)
+            torch.distributed.all_reduce(
+                packed_amaxes, op=torch.distributed.ReduceOp.MAX, group=self.data_parallel_group
+            )
+            _multi_tensor_copy_this_to_that(packed_amax_views, amaxes, dummy_overflow_buf)
+
     @torch.no_grad()
     def step_with_ready_grads(self) -> bool:
         """Step the optimizer with ready gradients, return successful.
@@ -1789,6 +2068,9 @@ def step_with_ready_grads(self) -> bool:
         """
         self.update_successful = super().step_with_ready_grads()
 
+        # If there is no FP8 parameters, this will do nothing.
+        self._update_fp8_scale_inv_and_amax()
+
         timers = self.config.timers
         if timers is not None:
             timers('params-all-gather', log_level=1).start(barrier=self.config.barrier_with_L1_time)
diff --git a/megatron/core/utils.py b/megatron/core/utils.py
index dcb1af833c..734755b8b1 100644
--- a/megatron/core/utils.py
+++ b/megatron/core/utils.py
@@ -1240,3 +1240,19 @@ def __exit__(
 __straggler__ = StragglerDetector()
 """StragglerDetector: private module variable, not be directly accessed
 """
+
+
+# Check if Transformer Engine has Float8Tensor class
+HAVE_TE_FLOAT8TENSOR = False
+try:
+    from transformer_engine.pytorch.float8_tensor import Float8Tensor
+
+    HAVE_TE_FLOAT8TENSOR = True
+except (ImportError, ModuleNotFoundError):
+    # Float8Tensor not found
+    pass
+
+
+def is_float8tensor(tensor: torch.Tensor) -> bool:
+    """Check if a tensor is a Transformer Engine Float8Tensor"""
+    return HAVE_TE_FLOAT8TENSOR and isinstance(tensor, Float8Tensor)
diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py
index 5ec39501c9..fa0a4fa76d 100644
--- a/megatron/training/arguments.py
+++ b/megatron/training/arguments.py
@@ -313,6 +313,10 @@ def validate_args(args, defaults={}):
         assert args.virtual_pipeline_model_parallel_size is not None, \
             '--align-param-gather only supported with interleaved pipeline parallelism'
 
+    if args.fp8_param_gather:
+        assert args.use_distributed_optimizer, \
+            '--fp8-param-gather only supported with distributed optimizer'
+
     # Parameters dtype.
     args.params_dtype = torch.float
     if args.fp16:
@@ -707,6 +711,9 @@ def _add_transformer_engine_args(parser):
     group.add_argument('--transformer-impl', default='transformer_engine',
                        choices=['local', 'transformer_engine'],
                        help='Which Transformer implementation to use.')
+    group.add_argument('--fp8-param-gather', action='store_true',
+                       help='Keep the compute param in fp8 (do not use any other intermediate '
+                            'dtype) and perform the param all-gather in fp8.')
 
     return parser
 
diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py
index fca80acc91..a0eef1f63c 100644
--- a/megatron/training/checkpointing.py
+++ b/megatron/training/checkpointing.py
@@ -21,6 +21,7 @@
 from megatron.core.dist_checkpointing.strategies.fully_parallel import \
     FullyParallelSaveStrategyWrapper, FullyParallelLoadStrategyWrapper
 from megatron.core.num_microbatches_calculator import update_num_microbatches
+from megatron.core.utils import is_float8tensor
 from .async_utils import schedule_async_save
 from .global_vars import get_args, get_one_logger
 from .utils import unwrap_model, print_rank_0, append_to_progress_log, is_last_rank
@@ -900,6 +901,20 @@ def _set_arg(arg_name, old_arg_name=None, force=False):
     return args, checkpoint_args
 
 
+def fix_fp8_params_lose_precision_when_loading_dist_ckpt(state_dict):
+    """
+    When "--fp8-param-gather" and "--use-dist-ckpt" are both enabled, the state dict read from
+    dist-checkpoint loses precision (the weights read from checkpoint go through the process of
+    bf16/fp16 -> fp8 -> bf16/fp16). This function is implemented to solve this problem.
+    When "--fp8-param-gather" is disabled, this function doesn't modify anything.
+    """
+    for key in state_dict.keys():
+        if key.startswith('model'):
+            for _, sharded_tensor in state_dict[key].items():
+                if is_float8tensor(sharded_tensor.data):
+                    sharded_tensor.data = sharded_tensor.data.from_float8().cpu()
+
+
 def load_checkpoint(model, optimizer, opt_param_scheduler, load_arg='load', strict=True,
                     ft_client=None):
     """Load a model checkpoint and return the iteration.
@@ -990,6 +1005,8 @@ def load_checkpoint(model, optimizer, opt_param_scheduler, load_arg='load', stri
                 gen_sd_opt_param_scheduler = None
             load_kwargs['sharded_state_dict'] = generate_state_dict(args, model, gen_sd_optim, gen_sd_opt_param_scheduler,
                                                                     gen_sd_rng_state, True, optim_sd_kwargs=optim_sd_kwargs)
+            # When "--fp8-param-gather" is disabled, this function doesn't modify anything.
+            fix_fp8_params_lose_precision_when_loading_dist_ckpt(load_kwargs['sharded_state_dict'])
 
     state_dict, checkpoint_name, release = _load_base_checkpoint(
         load_dir, args, rank0=False, **load_kwargs
diff --git a/megatron/training/training.py b/megatron/training/training.py
index b5f8b1ee10..a48accdb74 100644
--- a/megatron/training/training.py
+++ b/megatron/training/training.py
@@ -20,7 +20,12 @@
 import torch
 
 from megatron.core import mpu, tensor_parallel
-from megatron.core.utils import check_param_hashes_across_dp_replicas, get_model_config, StragglerDetector
+from megatron.core.utils import (
+    check_param_hashes_across_dp_replicas,
+    get_model_config,
+    StragglerDetector,
+    is_float8tensor,
+)
 from megatron.training.checkpointing import load_checkpoint
 from megatron.training.checkpointing import save_checkpoint
 from megatron.legacy.model import Float16Module
@@ -73,12 +78,13 @@
 
 stimer = StragglerDetector()
 
+
 def destroy_global_state():
     destroy_global_vars()
     destroy_num_microbatches_calculator()
     destroy_global_memory_buffer()
     destroy_model_parallel()
-    
+
 
 def print_datetime(string):
     """Note that this call will sync across all ranks."""
@@ -486,6 +492,21 @@ def get_model(model_provider_func, model_type=ModelType.encoder_or_decoder, wrap
     if args.fp16 or args.bf16:
         model = [Float16Module(model_module, args) for model_module in model]
 
+    # The model_module.bfloat16()/model_module.half() above will call the inplace copy of TE's
+    # Float8Tensor, which will write an unwanted value (amax calculated from the current fp8
+    # param) to its amax_history. The following logic will correct the amax_history back.
+    for model_module in model:
+        for param in model_module.parameters():
+            if is_float8tensor(param) and param._fp8_meta is not None:
+                fp8_meta = param._fp8_meta['scaling_fwd']
+                fp8_meta_index = param._fp8_meta_index
+                if hasattr(param, 'get_high_precision_init_val'):
+                    fp8_meta.amax_history[0][fp8_meta_index].copy_(
+                        param.get_high_precision_init_val().abs().max()
+                    )
+                else:
+                    fp8_meta.amax_history[0][fp8_meta_index] = 0
+
     if wrap_with_ddp:
         config = get_model_config(model[0])
         ddp_config = DistributedDataParallelConfig(
@@ -494,7 +515,8 @@ def get_model(model_provider_func, model_type=ModelType.encoder_or_decoder, wrap
             use_distributed_optimizer=args.use_distributed_optimizer,
             check_for_nan_in_grad=args.check_for_nan_in_loss_and_grad,
             bucket_size=args.ddp_bucket_size,
-            average_in_collective=args.ddp_average_in_collective)
+            average_in_collective=args.ddp_average_in_collective,
+            fp8_param_gather=args.fp8_param_gather)
         overlap_param_gather_with_optimizer_step = getattr(args, 'overlap_param_gather_with_optimizer_step', False)
         model = [DDP(config,
                      ddp_config,
@@ -625,7 +647,7 @@ def setup_model_and_optimizer(model_provider_func,
         args.ckpt_format = args.ckpt_convert_format
         args.save = os.path.join(args.ckpt_convert_save, args.ckpt_convert_format)
         update_use_dist_ckpt(args)
-        
+
         save_checkpoint(args.iteration, model, optimizer, opt_param_scheduler,
                         args.num_floating_point_operations_so_far)
 
diff --git a/pretrain_gpt.py b/pretrain_gpt.py
index 9658e0700f..d3be6df091 100644
--- a/pretrain_gpt.py
+++ b/pretrain_gpt.py
@@ -4,6 +4,8 @@
 import os
 import torch
 from functools import partial
+from contextlib import nullcontext
+import inspect
 
 from typing import Union
 from megatron.training import get_args
@@ -75,20 +77,36 @@ def model_provider(pre_process=True, post_process=True) -> Union[GPTModel, megat
             else:
                 transformer_layer_spec = get_gpt_layer_local_spec(args.num_experts, args.moe_grouped_gemm, args.qk_layernorm)
 
-        model = GPTModel(
-            config=config,
-            transformer_layer_spec=transformer_layer_spec,
-            vocab_size=args.padded_vocab_size,
-            max_sequence_length=args.max_position_embeddings,
-            pre_process=pre_process,
-            post_process=post_process,
-            fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
-            parallel_output=True,
-            share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
-            position_embedding_type=args.position_embedding_type,
-            rotary_percent=args.rotary_percent,
-            rotary_base=args.rotary_base
-        )
+        build_model_context = nullcontext
+        build_model_context_args = {}
+        if args.fp8_param_gather:
+            try:
+                from transformer_engine.pytorch import fp8_model_init
+
+                build_model_context = fp8_model_init
+                build_model_context_args["enabled"] = True
+
+                # Check if fp8_model_init supports preserve_high_precision_init_val
+                if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters:
+                    build_model_context_args["preserve_high_precision_init_val"] = True
+            except:
+                raise RuntimeError("--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found.")
+
+        with build_model_context(**build_model_context_args):
+            model = GPTModel(
+                config=config,
+                transformer_layer_spec=transformer_layer_spec,
+                vocab_size=args.padded_vocab_size,
+                max_sequence_length=args.max_position_embeddings,
+                pre_process=pre_process,
+                post_process=post_process,
+                fp16_lm_cross_entropy=args.fp16_lm_cross_entropy,
+                parallel_output=True,
+                share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights,
+                position_embedding_type=args.position_embedding_type,
+                rotary_percent=args.rotary_percent,
+                rotary_base=args.rotary_base
+            )
 
     return model
 
diff --git a/tests/functional_tests/jet_recipes/gpt.yaml b/tests/functional_tests/jet_recipes/gpt.yaml
index 87b5168fbb..559d748bc1 100644
--- a/tests/functional_tests/jet_recipes/gpt.yaml
+++ b/tests/functional_tests/jet_recipes/gpt.yaml
@@ -145,8 +145,5 @@ products:
     - gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp
     - gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp
     - gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp
+    - gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp
     - gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp
-
-    
-
-    
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/golden_values.json
new file mode 100644
index 0000000000..7335b2067c
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [18.28053, 0.49505, 0.49249, 0.4863, 0.49126, 0.48294, 0.48297, 0.49211, 0.49244, 0.48476, 0.49685, 0.48221, 0.48444, 0.48262, 0.4868, 0.4822, 0.48935, 0.49261, 0.49648, 0.48319, 0.48763, 0.48829, 0.48803, 0.48167, 0.48323, 0.48629, 0.48421, 0.48466, 0.48642, 0.48171, 0.5845, 0.48341, 0.47926, 0.48909, 0.49939, 0.50358, 0.4812, 0.48449, 0.48356, 0.48264, 0.48384, 0.48252, 0.4847, 0.48316, 0.48125, 0.48107, 0.57559, 0.48254, 0.48595, 0.48176, 0.48343, 0.48901, 0.48231, 0.48126, 0.48705, 0.48449, 0.48313, 0.48504, 0.49265, 0.49529, 0.48979, 0.48846, 0.48904, 0.48991, 0.49197, 0.48869, 0.48889, 0.49026, 0.49051, 0.48812, 0.4895, 0.4888, 0.49274, 0.49157, 0.49398, 0.68596, 0.48574, 0.48994, 0.48496, 0.496, 0.48608, 0.49521, 0.48726, 0.49274, 0.48836, 0.49429, 0.49013, 0.49126, 0.48792, 0.49147, 0.49169, 0.48964, 0.49008, 0.49378, 0.49365, 0.49165, 0.49075, 0.57694, 0.48973, 0.48945, 0.48773, 0.49186, 0.48699, 0.49202, 0.48785, 0.48984, 0.48807, 0.4924, 0.48739, 0.48901, 0.48669, 0.48864, 0.48892, 0.48906, 0.48729, 0.48907, 0.4886, 0.49334, 0.48702, 0.57734, 0.70083, 0.49192, 0.48993, 0.48756, 0.48839, 0.49692, 0.49292, 0.48647, 0.49172, 0.4875, 0.49397, 0.48663, 0.49145, 0.48815, 0.49401, 0.48878, 0.49212, 0.48753, 0.49235, 0.48811, 0.49451, 0.48865, 0.58524, 0.49262, 0.49011, 0.48923, 0.48823, 0.49108, 0.4881, 0.49074, 0.49805, 0.49124, 0.48831, 0.49161, 0.48613, 0.49324, 0.48948, 0.49372, 0.48427, 0.49263, 0.48691, 0.49317, 0.49667, 0.4969, 0.57482, 0.61619, 0.48773, 0.48884, 0.49076, 0.49017, 0.48952, 0.49239, 0.49075, 0.48963, 0.4911, 0.48939, 0.48983, 0.49046, 0.49409, 0.48869, 0.49044, 0.4872, 0.49356, 0.48711, 0.49475, 0.49335, 0.49242, 0.48938, 0.48799, 0.49308, 0.48649, 0.49513, 0.57985, 0.49149, 0.49028, 0.4911, 0.49172, 0.48942, 0.49435, 0.48938, 0.47502, 0.48947, 0.48882, 0.48685, 0.48977, 0.4839, 0.49208, 0.49183, 0.4899, 0.49107, 0.48954, 0.48936, 0.49081, 0.48809, 0.49012, 0.49118, 0.49592, 0.49005, 0.49234, 0.48935, 0.49702, 0.4881, 0.49255, 0.4923, 0.49215, 0.49408, 0.4896, 0.49166, 0.49036, 0.57641, 0.49203, 0.4866, 0.49827, 0.49306, 0.48826, 0.49197, 0.50213, 0.49344, 0.48736, 0.49635, 0.57884, 0.49438, 0.49181, 0.49665, 0.49267, 0.48679, 0.48884, 0.48977, 0.49284, 0.48791, 0.49204, 0.49178, 0.49595, 0.4931, 0.49191, 0.48826, 0.49306, 0.48701, 0.48992, 0.48579, 0.49069, 0.48562, 0.49508, 0.48592, 0.49748, 0.4852, 0.49001, 0.48851, 0.48928, 0.48685, 0.4898, 0.49343, 0.48889, 0.49276, 0.4874, 0.50472, 0.49085, 0.59958, 0.49141, 0.49279, 0.49191, 0.48975, 0.4895, 0.49082, 0.48927, 0.4914, 0.48634, 0.48671, 0.48679, 0.49495, 0.48847, 0.49036, 0.48784, 0.49319, 0.4893, 0.49337, 0.58198, 0.58629, 0.4953, 0.49089, 0.48763, 0.49392, 0.48743, 0.49484, 0.48893, 0.49356, 0.48948, 0.49182, 0.48987, 0.49043, 0.49529, 0.49039, 0.4921, 0.49072, 0.59678, 0.49229, 0.49187, 0.4928, 0.49741, 0.49468, 0.48644, 0.49313, 0.49332, 0.48749, 0.49394, 0.48779, 0.49346, 0.48849, 0.49244, 0.48985, 0.49183, 0.49358, 0.48865, 0.49267, 0.4914, 0.49166, 0.48871, 0.49327, 0.49077, 0.49024, 0.49629, 0.48853, 0.57947, 0.49147, 0.48886, 0.50383, 0.48817, 0.49188, 0.4873, 0.49974, 0.49014, 0.4908, 0.4922, 0.49589, 0.49266, 0.48782, 0.49383, 0.48872, 0.49176, 0.49069, 0.49264, 0.49042, 0.4914, 0.4912, 0.48803, 0.49078, 0.49007, 0.48811, 0.49406, 0.48945, 0.48976, 0.49052, 0.49238, 0.48839, 0.48749, 0.48884, 0.49154, 0.48706, 0.48761, 0.49108, 0.49077, 0.49131, 0.49425, 0.48822, 0.49246, 0.49172, 0.49273, 0.57851, 0.49276, 0.49599, 0.48901, 0.49655, 0.49128, 0.48808, 0.49162, 0.49012, 0.49189, 0.50308, 0.49552, 0.48646]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [9.21276, 0.28687, 0.28815, 0.2833, 0.28439, 0.27844, 0.27842, 0.28317, 0.28459, 0.28018, 0.29052, 0.27923, 0.27964, 0.27881, 0.28284, 0.27894, 0.2858, 0.28599, 0.29109, 0.28083, 0.28444, 0.28303, 0.2848, 0.27728, 0.28052, 0.2809, 0.27929, 0.2805, 0.28333, 0.27803, 0.3776, 0.27848, 0.27391, 0.28208, 0.29927, 0.30354, 0.28082, 0.28432, 0.28327, 0.28318, 0.28355, 0.28207, 0.28438, 0.28242, 0.28127, 0.28045, 0.37514, 0.2813, 0.28253, 0.28106, 0.28235, 0.28881, 0.28182, 0.28128, 0.28489, 0.28348, 0.2813, 0.28279, 0.29008, 0.29295, 0.28746, 0.2869, 0.28708, 0.28818, 0.28744, 0.28543, 0.28582, 0.28782, 0.28724, 0.28631, 0.28595, 0.28734, 0.2881, 0.28983, 0.2918, 0.48123, 0.28384, 0.28784, 0.28341, 0.28813, 0.28363, 0.29108, 0.2853, 0.28861, 0.28671, 0.29218, 0.28714, 0.29008, 0.28661, 0.29, 0.28895, 0.28724, 0.289, 0.29102, 0.28959, 0.28779, 0.28919, 0.37298, 0.28802, 0.28671, 0.28631, 0.29013, 0.28597, 0.29054, 0.28653, 0.28662, 0.28618, 0.28937, 0.285, 0.28745, 0.28473, 0.2862, 0.28623, 0.28613, 0.28465, 0.28674, 0.2875, 0.2909, 0.28626, 0.37409, 0.49531, 0.29025, 0.28653, 0.28605, 0.284, 0.29546, 0.29024, 0.28506, 0.29074, 0.28487, 0.29199, 0.28427, 0.28721, 0.28569, 0.28978, 0.28671, 0.29019, 0.2858, 0.29107, 0.28549, 0.28872, 0.28587, 0.38328, 0.28744, 0.28899, 0.28716, 0.28682, 0.28652, 0.28709, 0.28668, 0.29569, 0.28914, 0.28688, 0.28981, 0.28508, 0.29181, 0.28828, 0.29083, 0.28368, 0.28892, 0.28472, 0.2903, 0.29275, 0.29136, 0.3738, 0.41333, 0.28566, 0.28691, 0.28887, 0.2879, 0.28701, 0.2905, 0.28746, 0.28816, 0.28899, 0.28753, 0.2884, 0.28928, 0.29105, 0.28699, 0.28797, 0.28497, 0.29203, 0.28489, 0.28827, 0.29119, 0.29128, 0.28793, 0.28557, 0.29143, 0.28602, 0.29322, 0.37776, 0.28815, 0.28911, 0.28768, 0.28978, 0.2868, 0.2925, 0.28589, 0.27191, 0.28653, 0.28666, 0.28333, 0.28729, 0.28057, 0.28965, 0.2861, 0.28679, 0.28928, 0.28452, 0.28737, 0.28913, 0.28511, 0.28745, 0.28832, 0.29349, 0.28729, 0.28924, 0.28804, 0.29076, 0.28598, 0.29056, 0.28869, 0.28825, 0.29164, 0.28711, 0.28995, 0.2878, 0.37312, 0.28833, 0.28482, 0.29549, 0.28742, 0.28591, 0.28649, 0.29968, 0.29157, 0.2854, 0.29423, 0.37624, 0.29269, 0.28871, 0.29189, 0.28756, 0.28409, 0.28672, 0.28672, 0.29028, 0.28554, 0.29097, 0.28867, 0.29335, 0.29036, 0.28781, 0.28622, 0.28846, 0.28532, 0.28399, 0.28365, 0.28792, 0.28385, 0.29346, 0.28436, 0.29447, 0.28249, 0.28597, 0.28637, 0.28537, 0.28417, 0.28799, 0.28802, 0.28653, 0.29059, 0.28295, 0.30255, 0.28676, 0.39524, 0.28938, 0.28909, 0.28993, 0.28689, 0.2868, 0.28486, 0.2869, 0.28468, 0.28373, 0.28395, 0.28399, 0.29311, 0.28649, 0.28867, 0.2844, 0.29111, 0.28595, 0.29083, 0.37422, 0.38481, 0.2917, 0.28795, 0.28411, 0.29214, 0.28545, 0.29182, 0.28619, 0.29032, 0.28643, 0.28955, 0.287, 0.28693, 0.29048, 0.28673, 0.28964, 0.28608, 0.39417, 0.28909, 0.28926, 0.28892, 0.29626, 0.29035, 0.28418, 0.29096, 0.28911, 0.2861, 0.29247, 0.28616, 0.28914, 0.28625, 0.28976, 0.28808, 0.28866, 0.29068, 0.28692, 0.29086, 0.28868, 0.29004, 0.28595, 0.29148, 0.28842, 0.2886, 0.29171, 0.28773, 0.3764, 0.28898, 0.28636, 0.29892, 0.28549, 0.28973, 0.28465, 0.29697, 0.28725, 0.28663, 0.2894, 0.294, 0.29116, 0.28622, 0.29179, 0.28632, 0.29035, 0.28768, 0.28989, 0.28709, 0.2891, 0.28817, 0.28602, 0.28837, 0.28768, 0.28625, 0.28964, 0.28715, 0.287, 0.28748, 0.29025, 0.28485, 0.28473, 0.2867, 0.28777, 0.28402, 0.28515, 0.28793, 0.28644, 0.2893, 0.28758, 0.28612, 0.28687, 0.29012, 0.2871, 0.37328, 0.28876, 0.29273, 0.28732, 0.29333, 0.28722, 0.28605, 0.2878, 0.28786, 0.28733, 0.29635, 0.29189, 0.28435]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3.24795, 0.21194, 0.21471, 0.20869, 0.21204, 0.20759, 0.20377, 0.2107, 0.20945, 0.20618, 0.21705, 0.20521, 0.20785, 0.20627, 0.20635, 0.2064, 0.20649, 0.21053, 0.21523, 0.20491, 0.20938, 0.20895, 0.21121, 0.20684, 0.20811, 0.20914, 0.20848, 0.20944, 0.21029, 0.2088, 0.20823, 0.20765, 0.20786, 0.21144, 0.20746, 0.20856, 0.20791, 0.20961, 0.20962, 0.20803, 0.20624, 0.20748, 0.20646, 0.20637, 0.20506, 0.20636, 0.20873, 0.20709, 0.21021, 0.20645, 0.20725, 0.21067, 0.20689, 0.20484, 0.21018, 0.20758, 0.20809, 0.20663, 0.21735, 0.22092, 0.2181, 0.21664, 0.21604, 0.21705, 0.21811, 0.2175, 0.21613, 0.21894, 0.2186, 0.21706, 0.21821, 0.21776, 0.22265, 0.21862, 0.2187, 0.21766, 0.21611, 0.217, 0.21459, 0.22041, 0.21715, 0.2188, 0.21633, 0.21946, 0.21474, 0.21906, 0.21831, 0.21662, 0.21778, 0.21777, 0.21604, 0.21593, 0.21431, 0.21926, 0.2178, 0.21741, 0.21712, 0.22133, 0.2158, 0.21733, 0.21522, 0.21854, 0.21582, 0.21924, 0.21532, 0.21807, 0.216, 0.22003, 0.21598, 0.21559, 0.21655, 0.21799, 0.21734, 0.21749, 0.21785, 0.21759, 0.21855, 0.21936, 0.21602, 0.21592, 0.21786, 0.22091, 0.21874, 0.21753, 0.21923, 0.22306, 0.22024, 0.21591, 0.22007, 0.2187, 0.222, 0.2157, 0.22232, 0.21719, 0.22251, 0.21763, 0.22074, 0.21731, 0.21953, 0.21712, 0.22337, 0.22066, 0.22071, 0.21949, 0.21972, 0.21565, 0.21695, 0.22019, 0.21716, 0.219, 0.22553, 0.21923, 0.21738, 0.2203, 0.21678, 0.22028, 0.21797, 0.22029, 0.21479, 0.22065, 0.21605, 0.22109, 0.22372, 0.22023, 0.2184, 0.21646, 0.21673, 0.21835, 0.21624, 0.21877, 0.21593, 0.21993, 0.21906, 0.21748, 0.21846, 0.21846, 0.21773, 0.21782, 0.22154, 0.21764, 0.2193, 0.2172, 0.21983, 0.21556, 0.22293, 0.22107, 0.22132, 0.21857, 0.21717, 0.22128, 0.21593, 0.22043, 0.22094, 0.22038, 0.21956, 0.21936, 0.21966, 0.21754, 0.22141, 0.21803, 0.21648, 0.21739, 0.21902, 0.21686, 0.21805, 0.21493, 0.22077, 0.22186, 0.21962, 0.22048, 0.22052, 0.21855, 0.21913, 0.21681, 0.21996, 0.22012, 0.22218, 0.22009, 0.21986, 0.21939, 0.22266, 0.2163, 0.21865, 0.22182, 0.2197, 0.22192, 0.21676, 0.22102, 0.21734, 0.22013, 0.21984, 0.21564, 0.22434, 0.22271, 0.21673, 0.22212, 0.22818, 0.22064, 0.21733, 0.22214, 0.21857, 0.2223, 0.22007, 0.22387, 0.22019, 0.21548, 0.21818, 0.21601, 0.22079, 0.21586, 0.22149, 0.2206, 0.2192, 0.22065, 0.22097, 0.21714, 0.22179, 0.21621, 0.21994, 0.21491, 0.21991, 0.21504, 0.2197, 0.21388, 0.2201, 0.21487, 0.21828, 0.21636, 0.2175, 0.2155, 0.21587, 0.22018, 0.2151, 0.21983, 0.21588, 0.22793, 0.21875, 0.21694, 0.21987, 0.21989, 0.2186, 0.21826, 0.21718, 0.21971, 0.21741, 0.22031, 0.21565, 0.21643, 0.21559, 0.22115, 0.21694, 0.21849, 0.2154, 0.2201, 0.2167, 0.21944, 0.22561, 0.21402, 0.22049, 0.21782, 0.21537, 0.22116, 0.2162, 0.21949, 0.21494, 0.21795, 0.21647, 0.2181, 0.21867, 0.21751, 0.22266, 0.21692, 0.21888, 0.218, 0.22288, 0.21842, 0.21856, 0.21818, 0.22158, 0.22161, 0.21476, 0.21952, 0.21926, 0.21497, 0.21832, 0.21576, 0.21887, 0.2162, 0.21752, 0.21687, 0.21921, 0.22035, 0.21626, 0.22133, 0.21774, 0.22037, 0.21522, 0.22047, 0.21579, 0.21844, 0.22391, 0.21642, 0.21898, 0.21906, 0.21598, 0.22975, 0.21527, 0.21717, 0.21546, 0.22404, 0.21811, 0.21888, 0.2205, 0.22021, 0.22075, 0.21565, 0.21932, 0.21653, 0.21917, 0.21911, 0.22008, 0.21787, 0.21844, 0.21948, 0.21617, 0.21938, 0.21829, 0.21659, 0.2228, 0.21857, 0.21702, 0.21841, 0.21741, 0.21545, 0.21539, 0.21773, 0.21824, 0.21609, 0.21521, 0.21832, 0.21767, 0.21765, 0.21961, 0.21554, 0.21864, 0.21727, 0.21996, 0.21834, 0.21793, 0.22003, 0.21486, 0.22016, 0.21713, 0.21621, 0.21798, 0.21593, 0.21822, 0.22518, 0.21883, 0.21389]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.60577, 0.00374, 0.00393, 0.00334, 0.0036, 0.00342, 0.00344, 0.00397, 0.00331, 0.00323, 0.00356, 0.00332, 0.00341, 0.00356, 0.00347, 0.00308, 0.00337, 0.00327, 0.00342, 0.00359, 0.00317, 0.00312, 0.00326, 0.00315, 0.00321, 0.00318, 0.00314, 0.00309, 0.00313, 0.0031, 0.00327, 0.00314, 0.00303, 0.00338, 0.00311, 0.00306, 0.00302, 0.00321, 0.00306, 0.0032, 0.00305, 0.00309, 0.00302, 0.00328, 0.00297, 0.00295, 0.00322, 0.00301, 0.00307, 0.00325, 0.00287, 0.00312, 0.00289, 0.00302, 0.00308, 0.00307, 0.00308, 0.0035, 0.00327, 0.0032, 0.00318, 0.00312, 0.00322, 0.00336, 0.00333, 0.00345, 0.00311, 0.00326, 0.00307, 0.00318, 0.00309, 0.00331, 0.0031, 0.00327, 0.00333, 0.0033, 0.00321, 0.00328, 0.00317, 0.00325, 0.00309, 0.0033, 0.00326, 0.00323, 0.00321, 0.00319, 0.00318, 0.00329, 0.00315, 0.00331, 0.00368, 0.00361, 0.00377, 0.00374, 0.00383, 0.00345, 0.00348, 0.00347, 0.00339, 0.0035, 0.00312, 0.00344, 0.00325, 0.00318, 0.00318, 0.00323, 0.00328, 0.00331, 0.00329, 0.00318, 0.00327, 0.0032, 0.00317, 0.00314, 0.00313, 0.00316, 0.00327, 0.00348, 0.00319, 0.00309, 0.00338, 0.00315, 0.00347, 0.00335, 0.00315, 0.00314, 0.00339, 0.00316, 0.00323, 0.00311, 0.00331, 0.00317, 0.00311, 0.00316, 0.00317, 0.00314, 0.00323, 0.00319, 0.00311, 0.00328, 0.00326, 0.00315, 0.00319, 0.0035, 0.00303, 0.00311, 0.00331, 0.00334, 0.00314, 0.00323, 0.00345, 0.00325, 0.00319, 0.00322, 0.00331, 0.00339, 0.00342, 0.00343, 0.00335, 0.00349, 0.00338, 0.00342, 0.00327, 0.00325, 0.00331, 0.00327, 0.00328, 0.00325, 0.00321, 0.00326, 0.00324, 0.00346, 0.00329, 0.00347, 0.00325, 0.00327, 0.00322, 0.0032, 0.00311, 0.00307, 0.00322, 0.00303, 0.00312, 0.00323, 0.00329, 0.00312, 0.00323, 0.00323, 0.00307, 0.00315, 0.00324, 0.00314, 0.00308, 0.00308, 0.00313, 0.00322, 0.00318, 0.0032, 0.0032, 0.00322, 0.02747, 0.00304, 0.0031, 0.00322, 0.00309, 0.00303, 0.00319, 0.00304, 0.00319, 0.00315, 0.00305, 0.00324, 0.00328, 0.00297, 0.0033, 0.00302, 0.00329, 0.00319, 0.00309, 0.00319, 0.00324, 0.00336, 0.00317, 0.00324, 0.00322, 0.00343, 0.00323, 0.00314, 0.00337, 0.00333, 0.00319, 0.00305, 0.00351, 0.00342, 0.00323, 0.00333, 0.00325, 0.00329, 0.00309, 0.00337, 0.00313, 0.00331, 0.00309, 0.00329, 0.00319, 0.00325, 0.00323, 0.00324, 0.00332, 0.0034, 0.0033, 0.00322, 0.00318, 0.00319, 0.00329, 0.00315, 0.00329, 0.00325, 0.00333, 0.00322, 0.00337, 0.00313, 0.00313, 0.00327, 0.00332, 0.00313, 0.00307, 0.00312, 0.00306, 0.00322, 0.00309, 0.0033, 0.00323, 0.00341, 0.00326, 0.0035, 0.00329, 0.00341, 0.00333, 0.00334, 0.00347, 0.00314, 0.00336, 0.00336, 0.00329, 0.0032, 0.00322, 0.00331, 0.00337, 0.00336, 0.00312, 0.00321, 0.00407, 0.00319, 0.00353, 0.00339, 0.00344, 0.00327, 0.00338, 0.00335, 0.00325, 0.00334, 0.00318, 0.00329, 0.00329, 0.00323, 0.00318, 0.00325, 0.00322, 0.00317, 0.00327, 0.00307, 0.00322, 0.00305, 0.00323, 0.00318, 0.00328, 0.00317, 0.00326, 0.00313, 0.00312, 0.00317, 0.00319, 0.00322, 0.00326, 0.00311, 0.00318, 0.00349, 0.00314, 0.00329, 0.00324, 0.00339, 0.0031, 0.00326, 0.00308, 0.00316, 0.0031, 0.0034, 0.00318, 0.00327, 0.00321, 0.00313, 0.00335, 0.00311, 0.00333, 0.00329, 0.0031, 0.00325, 0.00325, 0.00326, 0.0033, 0.00323, 0.00315, 0.00321, 0.00322, 0.003, 0.00355, 0.00301, 0.00302, 0.00319, 0.00323, 0.0032, 0.00321, 0.0031, 0.00344, 0.00317, 0.0033, 0.00322, 0.00317, 0.00318, 0.00314, 0.00328, 0.0033, 0.0033, 0.0031, 0.00321, 0.0033, 0.00315, 0.00323, 0.00342, 0.00315, 0.00321, 0.00324, 0.00312, 0.00341, 0.00323, 0.00333, 0.00335, 0.00334, 0.00324, 0.00319, 0.00335, 0.00319, 0.0032, 0.00317, 0.0033, 0.00322, 0.00334, 0.0034, 0.00306]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.03213, 0.0015, 0.00156, 0.00153, 0.00152, 0.00153, 0.00156, 0.00153, 0.00152, 0.00153, 0.00155, 0.00152, 0.00157, 0.00153, 0.00155, 0.00153, 0.00153, 0.00151, 0.00155, 0.00153, 0.00154, 0.00152, 0.00154, 0.00153, 0.00155, 0.00154, 0.00154, 0.00154, 0.00154, 0.00153, 0.00156, 0.00152, 0.00152, 0.00153, 0.00156, 0.00153, 0.00153, 0.00155, 0.00153, 0.00152, 0.00154, 0.00155, 0.00155, 0.00152, 0.00152, 0.00153, 0.00154, 0.00153, 0.00154, 0.00152, 0.00154, 0.00154, 0.00155, 0.00153, 0.00156, 0.00154, 0.00156, 0.00153, 0.00156, 0.00151, 0.00154, 0.00153, 0.00156, 0.00151, 0.00156, 0.00155, 0.00155, 0.00152, 0.00155, 0.00152, 0.00154, 0.00153, 0.00156, 0.00153, 0.00154, 0.00154, 0.00156, 0.00154, 0.00155, 0.00155, 0.00155, 0.00153, 0.00154, 0.00152, 0.00155, 0.00154, 0.00156, 0.00153, 0.00153, 0.00153, 0.00155, 0.00154, 0.00155, 0.00153, 0.00154, 0.00153, 0.00155, 0.00153, 0.00154, 0.00152, 0.00155, 0.00152, 0.00155, 0.00154, 0.00155, 0.00154, 0.00155, 0.00153, 0.00154, 0.00152, 0.00155, 0.00153, 0.00153, 0.00154, 0.00154, 0.00151, 0.00155, 0.00153, 0.00156, 0.00153, 0.00155, 0.00154, 0.00156, 0.00156, 0.00155, 0.00154, 0.00155, 0.00153, 0.00152, 0.00153, 0.00155, 0.00154, 0.00155, 0.00154, 0.00154, 0.00154, 0.00155, 0.00151, 0.00152, 0.00153, 0.00153, 0.00151, 0.00153, 0.00154, 0.00156, 0.00155, 0.00157, 0.00154, 0.00156, 0.00154, 0.00155, 0.00151, 0.00154, 0.00153, 0.00154, 0.00153, 0.00156, 0.00155, 0.00155, 0.00152, 0.00157, 0.00153, 0.00154, 0.00154, 0.00155, 0.00154, 0.00151, 0.00154, 0.00155, 0.00152, 0.00155, 0.00152, 0.00156, 0.00153, 0.00153, 0.00155, 0.00154, 0.00153, 0.00154, 0.00152, 0.00154, 0.00155, 0.00154, 0.00152, 0.00157, 0.00154, 0.00154, 0.00152, 0.00155, 0.00152, 0.00157, 0.00152, 0.00154, 0.00153, 0.00156, 0.00153, 0.00156, 0.00154, 0.00156, 0.00153, 0.00154, 0.00153, 0.00157, 0.00155, 0.00154, 0.00156, 0.00154, 0.00153, 0.00151, 0.00156, 0.00156, 0.00155, 0.00155, 0.00154, 0.00155, 0.00154, 0.00155, 0.00152, 0.00154, 0.00154, 0.00154, 0.00156, 0.00157, 0.00154, 0.00155, 0.00155, 0.00153, 0.00153, 0.00154, 0.00155, 0.00155, 0.00155, 0.00155, 0.00154, 0.00154, 0.00154, 0.00154, 0.00153, 0.00154, 0.00154, 0.00154, 0.00154, 0.00155, 0.00154, 0.00156, 0.00156, 0.00154, 0.00155, 0.00153, 0.00155, 0.00152, 0.00156, 0.00154, 0.00156, 0.00156, 0.00152, 0.00154, 0.00153, 0.00153, 0.00155, 0.00154, 0.00157, 0.00154, 0.00153, 0.00157, 0.00155, 0.00156, 0.00155, 0.00157, 0.00155, 0.00155, 0.00153, 0.00156, 0.00158, 0.00155, 0.00155, 0.00157, 0.00153, 0.00155, 0.00154, 0.00155, 0.00153, 0.00155, 0.00155, 0.00154, 0.00151, 0.00154, 0.00156, 0.00156, 0.00155, 0.00155, 0.00155, 0.00155, 0.00153, 0.00155, 0.00156, 0.00154, 0.00155, 0.00153, 0.00155, 0.00155, 0.00153, 0.00154, 0.00154, 0.00156, 0.00156, 0.00155, 0.00155, 0.00154, 0.00153, 0.00155, 0.00155, 0.00155, 0.00154, 0.00153, 0.00154, 0.00154, 0.00155, 0.00156, 0.00156, 0.00156, 0.00156, 0.00156, 0.00156, 0.00155, 0.00155, 0.00154, 0.00156, 0.00154, 0.00156, 0.00155, 0.00154, 0.00156, 0.00154, 0.00153, 0.00155, 0.00152, 0.00156, 0.00151, 0.00155, 0.00154, 0.00155, 0.00155, 0.00156, 0.00153, 0.00155, 0.00154, 0.00156, 0.00154, 0.00154, 0.00154, 0.00155, 0.00155, 0.00155, 0.00153, 0.00155, 0.00154, 0.00154, 0.00155, 0.00156, 0.00153, 0.00153, 0.00154, 0.00155, 0.00153, 0.00154, 0.00155, 0.00154, 0.00154, 0.00155, 0.00155, 0.00155, 0.00153, 0.00155, 0.00154, 0.00157, 0.00156, 0.00153, 0.00157, 0.00157, 0.00156, 0.00157, 0.00154, 0.00155, 0.00157, 0.00155, 0.00155, 0.00153, 0.00153, 0.00152, 0.00154, 0.00155, 0.00155, 0.00154, 0.00153, 0.00155, 0.00154, 0.00155, 0.00155, 0.00155]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00038, 0.00024, 0.00024, 0.00015, 0.00015, 0.00016, 0.00015, 0.00016, 0.00015, 0.00013, 0.00013, 0.00015, 0.00015, 0.00013, 0.00015, 0.00013, 0.00015, 0.00013, 0.00015, 0.00015, 0.00013, 0.00015, 0.00013, 0.00015, 0.00013, 0.00014, 0.00013, 0.00013, 0.00015, 0.00013, 0.00015, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00016, 0.00013, 0.00013, 0.00013, 0.00015, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00025, 0.00018, 0.00018, 0.00019, 0.00018, 0.0003, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00019, 0.00018, 0.00019, 0.00019, 0.00019, 0.00018, 0.00019, 0.00019, 0.00019, 0.00021, 0.00018, 0.00021, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.0002, 0.00019, 0.00018, 0.00018, 0.00018, 0.00019, 0.00021, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00021, 0.00018, 0.00019, 0.00019, 0.00019, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.0002, 0.00018, 0.00021, 0.00019, 0.00018, 0.00018, 0.0002, 0.00023, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00019, 0.00021, 0.00018, 0.00018, 0.00021, 0.00018, 0.0002, 0.00018, 0.00018, 0.00021, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.0002, 0.00021, 0.00019, 0.00018, 0.00021, 0.00021, 0.00018, 0.00019, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00021, 0.00018, 0.00019, 0.00021, 0.00018, 0.00018, 0.00021, 0.00018, 0.00021, 0.00018, 0.00018, 0.00019, 0.00021, 0.00021, 0.00021, 0.00021, 0.00018, 0.00018, 0.00019, 0.00019, 0.00018, 0.0002, 0.00021, 0.00021, 0.0002, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.00021, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00021, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00021, 0.00018, 0.0002, 0.00018, 0.00018, 0.00018, 0.00021, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00021, 0.00019, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00021, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00019, 0.00018, 0.00018, 0.00019, 0.00021, 0.00019, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00021, 0.00018, 0.00018, 0.00021, 0.00019, 0.00019, 0.00019, 0.00021, 0.00023, 0.00018, 0.00021, 0.00019, 0.00018, 0.00021, 0.00019, 0.00019, 0.00019, 0.00019, 0.00018, 0.00019, 0.00019, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00022, 0.00021, 0.00018]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.62631, 0.00104, 0.00106, 0.00093, 0.00092, 0.00096, 0.00095, 0.00096, 0.00092, 0.00091, 0.0009, 0.00091, 0.00101, 0.00091, 0.00091, 0.0009, 0.0009, 0.0009, 0.00093, 0.00094, 0.0009, 0.00115, 0.0009, 0.00092, 0.00091, 0.00098, 0.00089, 0.00091, 0.00091, 0.0009, 0.00094, 0.0009, 0.00095, 0.00091, 0.00091, 0.0009, 0.0009, 0.00091, 0.00091, 0.00091, 0.00091, 0.00091, 0.00091, 0.00091, 0.00092, 0.0009, 0.00093, 0.00093, 0.00091, 0.00091, 0.00101, 0.00091, 0.0009, 0.0009, 0.0009, 0.00091, 0.00091, 0.00107, 0.00099, 0.001, 0.00101, 0.001, 0.00179, 0.001, 0.001, 0.00101, 0.0011, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.001, 0.001, 0.00101, 0.00109, 0.00106, 0.001, 0.001, 0.00102, 0.00101, 0.00102, 0.00109, 0.00101, 0.00104, 0.001, 0.00099, 0.00103, 0.00102, 0.001, 0.001, 0.00113, 0.00082, 0.00079, 0.0008, 0.001, 0.00102, 0.00105, 0.001, 0.001, 0.001, 0.00102, 0.00079, 0.00105, 0.00079, 0.00106, 0.0008, 0.00079, 0.00099, 0.00087, 0.00101, 0.0008, 0.00099, 0.00086, 0.00101, 0.00083, 0.00081, 0.001, 0.0008, 0.001, 0.00085, 0.00081, 0.001, 0.00079, 0.001, 0.00101, 0.001, 0.00079, 0.001, 0.00106, 0.001, 0.001, 0.00103, 0.00104, 0.00079, 0.00101, 0.00084, 0.00079, 0.0008, 0.0008, 0.00109, 0.00105, 0.00099, 0.0008, 0.00101, 0.00101, 0.00102, 0.00102, 0.0008, 0.00079, 0.00111, 0.00101, 0.00099, 0.0008, 0.001, 0.00108, 0.00107, 0.00103, 0.00103, 0.00084, 0.00105, 0.001, 0.00101, 0.001, 0.00101, 0.00101, 0.001, 0.00101, 0.00101, 0.00114, 0.00099, 0.0008, 0.00079, 0.00101, 0.001, 0.001, 0.00105, 0.00101, 0.001, 0.00113, 0.00101, 0.001, 0.00106, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00106, 0.00105, 0.00107, 0.00106, 0.00102, 0.001, 0.00104, 0.00101, 0.00105, 0.001, 0.00104, 0.00105, 0.00104, 0.00103, 0.001, 0.001, 0.001, 0.00109, 0.00101, 0.00104, 0.001, 0.00108, 0.00108, 0.001, 0.00101, 0.001, 0.00103, 0.00106, 0.00102, 0.00106, 0.00102, 0.00099, 0.00101, 0.00105, 0.00104, 0.00101, 0.00105, 0.00102, 0.00103, 0.00102, 0.001, 0.001, 0.00104, 0.001, 0.00101, 0.00101, 0.001, 0.00105, 0.00101, 0.00107, 0.00102, 0.001, 0.00101, 0.00101, 0.00101, 0.00108, 0.00101, 0.001, 0.00106, 0.00101, 0.001, 0.001, 0.00105, 0.00101, 0.00116, 0.00112, 0.00101, 0.001, 0.00103, 0.00101, 0.00103, 0.00101, 0.00105, 0.00103, 0.00102, 0.001, 0.00101, 0.001, 0.00108, 0.00108, 0.00101, 0.00106, 0.00109, 0.00106, 0.00102, 0.00104, 0.001, 0.001, 0.00099, 0.00101, 0.00101, 0.001, 0.001, 0.001, 0.00102, 0.00105, 0.001, 0.00103, 0.00103, 0.001, 0.00101, 0.001, 0.00107, 0.00101, 0.001, 0.001, 0.00102, 0.001, 0.00111, 0.001, 0.00102, 0.00104, 0.00099, 0.001, 0.00101, 0.00101, 0.00105, 0.00101, 0.001, 0.00101, 0.00107, 0.00113, 0.00103, 0.00105, 0.00102, 0.00105, 0.00101, 0.00101, 0.00102, 0.001, 0.00101, 0.00103, 0.001, 0.00102, 0.00108, 0.00103, 0.00103, 0.00101, 0.00104, 0.001, 0.00103, 0.00101, 0.00107, 0.00106, 0.00099, 0.00103, 0.00102, 0.00101, 0.00102, 0.001, 0.00101, 0.00101, 0.00102, 0.001, 0.00101, 0.0011, 0.00101, 0.001, 0.00101, 0.001, 0.00108, 0.001, 0.0011, 0.00108, 0.00101, 0.001, 0.00102, 0.00102, 0.00101, 0.001, 0.00102, 0.00108, 0.00101, 0.00103, 0.001, 0.00101, 0.00101, 0.001, 0.00109, 0.001, 0.001, 0.00105, 0.00101, 0.00105, 0.001, 0.00102, 0.0011, 0.00103, 0.00103, 0.00102, 0.00106, 0.00104, 0.00104, 0.00107, 0.00101, 0.001, 0.00111, 0.00102, 0.00101, 0.00103, 0.00101, 0.00102, 0.001, 0.00102, 0.00103, 0.00101, 0.00101, 0.0011, 0.001, 0.00105, 0.00106, 0.00101]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00488, 0.00438, 0.00439, 0.00461, 0.00443, 0.0046, 0.00465, 0.00446, 0.00441, 0.00439, 0.00443, 0.0044, 0.00516, 0.00445, 0.0044, 0.0044, 0.00439, 0.0044, 0.0044, 0.00441, 0.00443, 0.00441, 0.00443, 0.00439, 0.00443, 0.0051, 0.0044, 0.00439, 0.00443, 0.00441, 0.0044, 0.00438, 0.00442, 0.00442, 0.00442, 0.00442, 0.00443, 0.0044, 0.00442, 0.00439, 0.0045, 0.00441, 0.00439, 0.00439, 0.0044, 0.00441, 0.00438, 0.00441, 0.00441, 0.0044, 0.00485, 0.00441, 0.00442, 0.00439, 0.0044, 0.00438, 0.00445, 0.00462, 0.00437, 0.00439, 0.0044, 0.00439, 0.0044, 0.00442, 0.00439, 0.00441, 0.00442, 0.00439, 0.00439, 0.00439, 0.00442, 0.0044, 0.00439, 0.00441, 0.00438, 0.00523, 0.00508, 0.00442, 0.00437, 0.00496, 0.00442, 0.00437, 0.00556, 0.00439, 0.00438, 0.00443, 0.00439, 0.0044, 0.00439, 0.00442, 0.00441, 0.0052, 0.00441, 0.00441, 0.00438, 0.00444, 0.00441, 0.0044, 0.00441, 0.00439, 0.00443, 0.00439, 0.00438, 0.00443, 0.0044, 0.00439, 0.00442, 0.00443, 0.00439, 0.00439, 0.00441, 0.00441, 0.0044, 0.00544, 0.00439, 0.0044, 0.0044, 0.00442, 0.00441, 0.00438, 0.00439, 0.00441, 0.00442, 0.00439, 0.00438, 0.00441, 0.00442, 0.0044, 0.0044, 0.00441, 0.00436, 0.0044, 0.00438, 0.00442, 0.00442, 0.00442, 0.00444, 0.00442, 0.00441, 0.0044, 0.00439, 0.00439, 0.00439, 0.00441, 0.00441, 0.00443, 0.00439, 0.00439, 0.00439, 0.00439, 0.00438, 0.0044, 0.00439, 0.00441, 0.00441, 0.00481, 0.00443, 0.0044, 0.0044, 0.00442, 0.0044, 0.00439, 0.0044, 0.00438, 0.00454, 0.0044, 0.00439, 0.0044, 0.00439, 0.0044, 0.0044, 0.00438, 0.00441, 0.00437, 0.00439, 0.0044, 0.00441, 0.00438, 0.00441, 0.00439, 0.00441, 0.00442, 0.0044, 0.00439, 0.00438, 0.00441, 0.00439, 0.00441, 0.0044, 0.0044, 0.0044, 0.00439, 0.0044, 0.00442, 0.00467, 0.00439, 0.0044, 0.0044, 0.00442, 0.00441, 0.00442, 0.0044, 0.00442, 0.00442, 0.00441, 0.00509, 0.00443, 0.0044, 0.00442, 0.00438, 0.00487, 0.00531, 0.00442, 0.00442, 0.00442, 0.00442, 0.00441, 0.00439, 0.00441, 0.0044, 0.00439, 0.0044, 0.00441, 0.00439, 0.00439, 0.0044, 0.0044, 0.00439, 0.00443, 0.00441, 0.00454, 0.00439, 0.00441, 0.0044, 0.00441, 0.00439, 0.00441, 0.00442, 0.0044, 0.00441, 0.00438, 0.0044, 0.00439, 0.0044, 0.0044, 0.00442, 0.0044, 0.0044, 0.0044, 0.00438, 0.0044, 0.0044, 0.0044, 0.0044, 0.0044, 0.00441, 0.00441, 0.0044, 0.00442, 0.0044, 0.00439, 0.00439, 0.00439, 0.00439, 0.00439, 0.0044, 0.00442, 0.00441, 0.00439, 0.00443, 0.00439, 0.0044, 0.0044, 0.00439, 0.0044, 0.0044, 0.00441, 0.0044, 0.00438, 0.00441, 0.00442, 0.0044, 0.00439, 0.00443, 0.00534, 0.00438, 0.00442, 0.0044, 0.0044, 0.00441, 0.00495, 0.00439, 0.00441, 0.00438, 0.00441, 0.00441, 0.0044, 0.00437, 0.00441, 0.00439, 0.0044, 0.00442, 0.0044, 0.00442, 0.00439, 0.00437, 0.00441, 0.0044, 0.00439, 0.0044, 0.00457, 0.00441, 0.00441, 0.00442, 0.00441, 0.00443, 0.00439, 0.00443, 0.00439, 0.00439, 0.00439, 0.00441, 0.00486, 0.00439, 0.00441, 0.00441, 0.00453, 0.0044, 0.00437, 0.00441, 0.0044, 0.00442, 0.0044, 0.00442, 0.00441, 0.00441, 0.00439, 0.00439, 0.00441, 0.00438, 0.0044, 0.00442, 0.00443, 0.0044, 0.0044, 0.00442, 0.00441, 0.00439, 0.00442, 0.00441, 0.0044, 0.00439, 0.00438, 0.00439, 0.00442, 0.00439, 0.00441, 0.00439, 0.0044, 0.00441, 0.0044, 0.00442, 0.00443, 0.0044, 0.00438, 0.0044, 0.00439, 0.00444, 0.00439, 0.00442, 0.0044, 0.00439, 0.00441, 0.00439, 0.00442, 0.00439, 0.00438, 0.00439, 0.00438, 0.0044, 0.00442, 0.0044, 0.00438, 0.00442, 0.00443, 0.0044, 0.0044, 0.00439, 0.00441, 0.00439, 0.0044, 0.00444, 0.00455, 0.00442, 0.00443, 0.00441, 0.00442, 0.00442, 0.00443, 0.0044]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00313, 0.00096, 0.00097, 0.00093, 0.00094, 0.00094, 0.00094, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00094, 0.00092, 0.00093, 0.00092, 0.00094, 0.00092, 0.00093, 0.00093, 0.00093, 0.00092, 0.00092, 0.00092, 0.00092, 0.00094, 0.00092, 0.00093, 0.00093, 0.00092, 0.00092, 0.00092, 0.00092, 0.00093, 0.00092, 0.00092, 0.00092, 0.00099, 0.00092, 0.00093, 0.00094, 0.00093, 0.00092, 0.00092, 0.00092, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00092, 0.00092, 0.00092, 0.00092, 0.00092, 0.00092, 0.00096, 0.00093, 0.00093, 0.00092, 0.00093, 0.00092, 0.00092, 0.00092, 0.00094, 0.00093, 0.00092, 0.00093, 0.00092, 0.00093, 0.00093, 0.00092, 0.00093, 0.00092, 0.00097, 0.00095, 0.00092, 0.00093, 0.00093, 0.00092, 0.00099, 0.00095, 0.00093, 0.00094, 0.00093, 0.00092, 0.00093, 0.00092, 0.00093, 0.00094, 0.00095, 0.00093, 0.00093, 0.00093, 0.00092, 0.00092, 0.00093, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00094, 0.00095, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00093, 0.00092, 0.00092, 0.00093, 0.00092, 0.00094, 0.00093, 0.00093, 0.00092, 0.00092, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00092, 0.00092, 0.00093, 0.00093, 0.00092, 0.00092, 0.00094, 0.00093, 0.00092, 0.00093, 0.00094, 0.00094, 0.00092, 0.00093, 0.00092, 0.00093, 0.00092, 0.00093, 0.00092, 0.00093, 0.00093, 0.00092, 0.00092, 0.00093, 0.00092, 0.00093, 0.00094, 0.00093, 0.00092, 0.00094, 0.00093, 0.00093, 0.00092, 0.00093, 0.00095, 0.00093, 0.00092, 0.00092, 0.00093, 0.00094, 0.00092, 0.00092, 0.00093, 0.00093, 0.00093, 0.00092, 0.00094, 0.00094, 0.00092, 0.00094, 0.00092, 0.00093, 0.00093, 0.00092, 0.00093, 0.00092, 0.00093, 0.00092, 0.00092, 0.00094, 0.00093, 0.00093, 0.00092, 0.00092, 0.00093, 0.00092, 0.00092, 0.00093, 0.00094, 0.00093, 0.00094, 0.00093, 0.00093, 0.00093, 0.00093, 0.00095, 0.00092, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00092, 0.00094, 0.00093, 0.00095, 0.00094, 0.00094, 0.00092, 0.00093, 0.00093, 0.00094, 0.00093, 0.00093, 0.00094, 0.00092, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00097, 0.00093, 0.00092, 0.00094, 0.00092, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00094, 0.00094, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00094, 0.00092, 0.00094, 0.00093, 0.00092, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00092, 0.00095, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00094, 0.00093, 0.00092, 0.00094, 0.00093, 0.00093, 0.00093, 0.00092, 0.00094, 0.00093, 0.00093, 0.00092, 0.00093, 0.00094, 0.00093, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00092, 0.00092, 0.00093, 0.00094, 0.00093, 0.00094, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00093, 0.00092, 0.00094, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00092, 0.00092, 0.00092, 0.00093, 0.00093, 0.00093, 0.00093, 0.00092, 0.00093, 0.00093, 0.00094, 0.00094, 0.00093, 0.00093, 0.00093, 0.00094, 0.00092, 0.00093, 0.00093, 0.00094, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00094, 0.00093, 0.00094, 0.00095, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00093, 0.00096, 0.00093, 0.00093, 0.00093, 0.00093, 0.00094, 0.00094, 0.00094]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0012, 0.001, 0.00119, 0.00096, 0.00096, 0.00096, 0.00097, 0.00096, 0.00096, 0.00096, 0.00095, 0.00096, 0.00097, 0.00095, 0.00096, 0.00096, 0.00096, 0.00096, 0.00096, 0.00096, 0.00097, 0.00096, 0.00096, 0.00095, 0.00096, 0.00097, 0.00096, 0.00095, 0.00096, 0.00096, 0.00096, 0.00096, 0.00096, 0.00095, 0.00095, 0.00095, 0.00096, 0.00104, 0.00096, 0.00095, 0.00097, 0.00095, 0.00096, 0.00096, 0.00096, 0.00096, 0.00096, 0.00095, 0.00096, 0.00096, 0.00097, 0.00096, 0.00096, 0.00095, 0.00096, 0.00095, 0.00096, 0.001, 0.00099, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.00098, 0.00098, 0.00098, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00099, 0.001, 0.00098, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00098, 0.00098, 0.00099, 0.00099, 0.00098, 0.00103, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.001, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00103, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.001, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00098, 0.001, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.00099, 0.00099, 0.00098, 0.00098, 0.00099, 0.00098, 0.00098, 0.001, 0.001, 0.001, 0.00099, 0.001, 0.001, 0.00099, 0.00099, 0.00099, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.001, 0.00099, 0.00098, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00102, 0.00099, 0.00099, 0.00098, 0.001, 0.00099, 0.00099, 0.001, 0.00099, 0.00098, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.00099, 0.00101, 0.00099, 0.00099, 0.00099, 0.00101, 0.00099, 0.00099, 0.00099, 0.001, 0.00098, 0.001, 0.00099, 0.001, 0.00099, 0.00101, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00101, 0.00099, 0.001, 0.00098, 0.00099, 0.00105, 0.00099, 0.00099, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00102, 0.00098, 0.00098, 0.00099, 0.001, 0.00099, 0.001, 0.001, 0.001, 0.00098, 0.00101, 0.00099, 0.001, 0.00098, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00098, 0.00101, 0.00099, 0.00098, 0.00099, 0.00103, 0.00098, 0.00099, 0.00099, 0.001, 0.00098, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00106, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00101, 0.001, 0.00099, 0.001, 0.001, 0.001, 0.00098, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.001, 0.00101, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.00101, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.001, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.00101, 0.001, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00098, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.001, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.001, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.001, 0.001, 0.00099, 0.00099, 0.001, 0.00101, 0.00099]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.63786, 0.00795, 0.00821, 0.00789, 0.00772, 0.00795, 0.00797, 0.00777, 0.00768, 0.00764, 0.00767, 0.00766, 0.0086, 0.00767, 0.00766, 0.00763, 0.00766, 0.00763, 0.00768, 0.0077, 0.00769, 0.0079, 0.00766, 0.00765, 0.00767, 0.00848, 0.00762, 0.00762, 0.0077, 0.00763, 0.0077, 0.0076, 0.00769, 0.00767, 0.00763, 0.00763, 0.00766, 0.0078, 0.00766, 0.00762, 0.00777, 0.00763, 0.00763, 0.00761, 0.00765, 0.00763, 0.00767, 0.00766, 0.00766, 0.00764, 0.00825, 0.00763, 0.00764, 0.00762, 0.00762, 0.00761, 0.00768, 0.00821, 0.00776, 0.00779, 0.00781, 0.00778, 0.00875, 0.00781, 0.00783, 0.00782, 0.00792, 0.00779, 0.00782, 0.00781, 0.00783, 0.00781, 0.0078, 0.00782, 0.0078, 0.00884, 0.00896, 0.00783, 0.00778, 0.00843, 0.00783, 0.00789, 0.00911, 0.0078, 0.00787, 0.00783, 0.00779, 0.00784, 0.00781, 0.00784, 0.00782, 0.00886, 0.00764, 0.00763, 0.00759, 0.00785, 0.00785, 0.0079, 0.00781, 0.0078, 0.00787, 0.00782, 0.00759, 0.00793, 0.00762, 0.00785, 0.00763, 0.00765, 0.00781, 0.00773, 0.00784, 0.00762, 0.0078, 0.00885, 0.00779, 0.00767, 0.00763, 0.00782, 0.00761, 0.0078, 0.00773, 0.00766, 0.00783, 0.00758, 0.00778, 0.00785, 0.00781, 0.00759, 0.00779, 0.00791, 0.00776, 0.0078, 0.00782, 0.0079, 0.00761, 0.00781, 0.00773, 0.0076, 0.00764, 0.0076, 0.0079, 0.00789, 0.00777, 0.00763, 0.00782, 0.00784, 0.00781, 0.00782, 0.00757, 0.0076, 0.00788, 0.0078, 0.00778, 0.00762, 0.0078, 0.00834, 0.00794, 0.00785, 0.00783, 0.00773, 0.0079, 0.0078, 0.00783, 0.0078, 0.00801, 0.00782, 0.0078, 0.0078, 0.00781, 0.00801, 0.00781, 0.00758, 0.0076, 0.00778, 0.00779, 0.0078, 0.00791, 0.00781, 0.00781, 0.00797, 0.00782, 0.00782, 0.0079, 0.0078, 0.00784, 0.00783, 0.00781, 0.00782, 0.00788, 0.0079, 0.00791, 0.0079, 0.00782, 0.00781, 0.00814, 0.0078, 0.00785, 0.00782, 0.00793, 0.00792, 0.008, 0.00785, 0.00786, 0.00784, 0.00782, 0.00866, 0.00784, 0.00789, 0.00784, 0.00787, 0.00839, 0.0088, 0.00783, 0.00783, 0.00785, 0.00793, 0.00785, 0.0079, 0.00785, 0.0078, 0.00782, 0.00791, 0.00786, 0.00781, 0.0079, 0.00782, 0.00783, 0.00783, 0.00783, 0.00782, 0.00798, 0.00781, 0.00795, 0.00782, 0.00782, 0.00791, 0.00782, 0.00789, 0.00781, 0.00782, 0.00779, 0.00782, 0.00781, 0.00795, 0.00784, 0.00781, 0.00787, 0.00782, 0.00781, 0.0078, 0.00791, 0.00784, 0.00796, 0.00798, 0.00782, 0.00782, 0.00785, 0.00784, 0.00818, 0.00781, 0.00787, 0.00783, 0.00781, 0.0078, 0.00782, 0.00781, 0.00794, 0.00793, 0.0078, 0.00794, 0.00789, 0.00786, 0.00784, 0.0079, 0.00782, 0.00783, 0.00781, 0.00784, 0.00779, 0.00782, 0.00783, 0.00781, 0.00781, 0.00789, 0.00881, 0.00824, 0.00789, 0.00781, 0.00781, 0.0078, 0.0085, 0.00783, 0.00782, 0.00779, 0.00783, 0.0078, 0.00797, 0.00779, 0.00784, 0.00789, 0.00782, 0.00783, 0.00779, 0.00782, 0.00789, 0.00779, 0.00783, 0.00781, 0.00786, 0.00799, 0.00801, 0.0079, 0.00782, 0.00791, 0.00782, 0.00785, 0.00781, 0.00784, 0.00782, 0.00783, 0.00779, 0.00783, 0.0084, 0.00783, 0.00791, 0.00782, 0.00798, 0.00782, 0.0078, 0.00782, 0.00787, 0.00792, 0.0078, 0.00787, 0.00784, 0.00783, 0.00784, 0.00779, 0.00783, 0.00781, 0.00782, 0.00783, 0.00786, 0.00794, 0.00785, 0.00783, 0.00782, 0.00781, 0.00795, 0.00782, 0.00795, 0.00789, 0.00781, 0.00783, 0.00785, 0.00782, 0.00782, 0.0078, 0.00782, 0.00794, 0.00782, 0.00786, 0.00785, 0.00783, 0.0078, 0.00783, 0.0079, 0.00784, 0.00781, 0.00787, 0.00781, 0.0079, 0.00782, 0.00782, 0.00796, 0.00784, 0.00782, 0.00783, 0.00789, 0.00792, 0.00787, 0.00791, 0.00781, 0.00783, 0.00802, 0.00784, 0.00783, 0.00785, 0.00783, 0.00782, 0.00781, 0.00788, 0.00802, 0.00787, 0.00787, 0.00793, 0.00784, 0.00793, 0.00797, 0.00783]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.88345, 10.90291, 10.88739, 10.83435, 10.68106, 10.65239, 10.43882, 10.15796, 9.94566, 9.85031, 9.59624, 9.85805, 9.88827, 9.63311, 9.79091, 9.51415, 9.46112, 9.65226, 9.38851, 9.33535, 9.24597, 9.15002, 9.1791, 9.00048, 9.19456, 9.06645, 9.16089, 9.17249, 9.30644, 8.99568, 8.93903, 9.04853, 9.05134, 8.65891, 8.72191, 8.75857, 8.68509, 8.7367, 8.66155, 8.76648, 8.66383, 8.85312, 8.83506, 8.49989, 8.39023, 8.43268, 8.49362, 8.38495, 8.4346, 8.58278, 8.36836, 8.19768, 8.22999, 8.22623, 8.27021, 7.91926, 8.10177, 7.89448, 8.24737, 8.23304, 8.007, 7.96876, 7.92354, 7.74219, 7.74672, 7.64691, 7.51972, 7.90702, 7.70393, 7.45184, 7.74158, 7.77006, 7.54684, 7.30265, 7.45642, 7.33883, 7.46797, 7.22942, 7.63514, 7.28131, 7.35335, 7.21286, 7.21895, 7.42346, 7.17843, 7.28509, 7.00192, 7.0089, 7.04286, 7.14056, 6.82835, 6.99014, 7.09279, 7.00447, 6.88003, 6.761, 6.99471, 7.0633, 6.70925, 6.5917, 6.73258, 6.74964, 6.73779, 6.74258, 6.66376, 6.41582, 6.64124, 6.62873, 6.45047, 6.63243, 6.75424, 6.61807, 6.73736, 6.70363, 6.63926, 6.51953, 6.61425, 6.42312, 6.67885, 6.26757, 6.26882, 6.32005, 6.41287, 6.37101, 6.46896, 6.31397, 6.36148, 6.25486, 6.22526, 6.42692, 6.35485, 6.35029, 6.19105, 6.18567, 6.26859, 6.415, 6.23334, 6.18337, 6.21035, 6.14535, 6.09626, 6.10387, 6.28772, 6.43606, 6.29503, 6.335, 6.13464, 6.21503, 6.02829, 6.06095, 5.9935, 6.28273, 6.22023, 5.99847, 5.81393, 6.16265, 5.87946, 6.14445, 5.82485, 6.19248, 6.18157, 6.12584, 5.97074, 6.14877, 5.98325, 6.23524, 5.93942, 5.83892, 5.82229, 5.72934, 6.05496, 6.0434, 6.11051, 5.93954, 6.09171, 6.01241, 6.04004, 6.0322, 5.99651, 5.89061, 6.00653, 5.67122, 5.75784, 5.94696, 5.9005, 5.91468, 5.82189, 5.89471, 5.77842, 5.61622, 5.78054, 5.69253, 5.90048, 5.66647, 5.77352, 5.78152, 5.97131, 5.71328, 5.92696, 5.81669, 5.94504, 5.4175, 5.97213, 5.95642, 5.93165, 5.48932, 5.49949, 5.70719, 5.6873, 5.5725, 5.66702, 5.76913, 5.57229, 5.82826, 5.61559, 5.69173, 5.731, 5.73072, 5.62169, 5.71676, 5.78883, 5.80232, 5.67949, 5.77122, 5.47901, 5.79612, 5.73059, 5.53929, 5.69307, 5.7447, 5.6605, 5.44825, 5.66038, 5.60993, 5.60208, 5.50359, 5.67847, 5.72987, 5.52511, 5.65798, 5.63632, 5.4706, 5.64734, 5.55245, 5.58744, 5.44937, 5.20181, 5.63792, 5.72045, 5.87194, 5.56238, 5.74796, 5.79022, 5.38902, 5.44605, 5.54282, 5.55739, 5.49575, 5.64498, 5.33577, 5.45876, 5.42673, 5.5365, 5.42129, 5.62761, 5.71678, 5.48104, 5.60527, 5.5126, 5.25058, 5.49118, 5.43681, 5.48508, 5.28923, 5.46474, 5.45286, 5.6724, 5.35082, 5.46484, 5.40053, 5.54964, 5.16851, 5.10998, 5.5302, 5.59551, 5.43932, 5.53394, 5.2946, 5.37074, 5.47423, 5.2811, 5.46993, 5.28979, 5.57821, 5.48542, 5.37281, 5.45382, 5.27315, 5.53883, 5.2931, 5.25971, 5.35796, 5.33386, 5.5094, 5.38011, 5.51219, 5.30068, 5.34103, 5.49541, 5.54901, 5.50235, 5.43059, 5.39677, 5.52711, 5.19094, 5.45817, 5.34325, 5.56956, 5.41302, 5.43584, 5.37612, 5.25951, 5.25447, 5.49422, 5.5781, 5.35768, 5.3279, 5.19136, 5.4016, 5.39747, 5.20526, 5.61362, 5.29418, 5.39709, 5.44712, 5.30146, 5.34724, 5.36676, 5.28901, 5.361, 5.45905, 5.27649, 5.47318, 5.21725, 5.22023, 5.35122, 5.28396, 5.21834, 5.10071, 5.23602, 5.43096, 5.33142, 5.33017, 5.66246, 5.3004, 5.30692, 5.39386, 5.13475, 5.06957, 5.3365, 5.37793, 5.21244, 5.29887, 5.36995, 5.34675, 5.15473, 5.24757, 5.27856, 5.16172, 5.08869, 5.37568, 5.11393, 5.55309, 5.15317, 5.32295, 5.06795, 5.13265, 5.17242, 5.01042, 5.01637, 5.20515, 5.17193, 5.18392, 5.30507, 5.25233, 5.31569, 5.14154, 5.24356, 5.12106, 5.31092, 5.36465, 5.24729, 5.09639, 5.1804, 5.29568, 5.10464, 5.27827, 5.10619, 5.10892, 5.03572]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.88345, 10.90291, 10.88739, 10.83435, 10.68106, 10.65239, 10.43882, 10.15796, 9.94566, 9.85031, 9.59624, 9.85805, 9.88827, 9.63311, 9.79091, 9.51415, 9.46112, 9.65226, 9.38851, 9.33535, 9.24597, 9.15002, 9.1791, 9.00048, 9.19456, 9.06645, 9.16089, 9.17249, 9.30644, 8.99568, 8.93903, 9.04853, 9.05134, 8.65891, 8.72191, 8.75857, 8.68509, 8.7367, 8.66155, 8.76648, 8.66383, 8.85312, 8.83506, 8.49989, 8.39023, 8.43268, 8.49362, 8.38495, 8.4346, 8.58278, 8.36836, 8.19768, 8.22999, 8.22623, 8.27021, 7.91926, 8.10177, 7.89448, 8.24737, 8.23304, 8.007, 7.96876, 7.92354, 7.74219, 7.74672, 7.64691, 7.51972, 7.90702, 7.70393, 7.45184, 7.74158, 7.77006, 7.54684, 7.30265, 7.45642, 7.33883, 7.46797, 7.22942, 7.63514, 7.28131, 7.35335, 7.21286, 7.21895, 7.42346, 7.17843, 7.28509, 7.00192, 7.0089, 7.04286, 7.14056, 6.82835, 6.99014, 7.09279, 7.00447, 6.88003, 6.761, 6.99471, 7.0633, 6.70925, 6.5917, 6.73258, 6.74964, 6.73779, 6.74258, 6.66376, 6.41582, 6.64124, 6.62873, 6.45047, 6.63243, 6.75424, 6.61807, 6.73736, 6.70363, 6.63926, 6.51953, 6.61425, 6.42312, 6.67885, 6.26757, 6.26882, 6.32005, 6.41287, 6.37101, 6.46896, 6.31397, 6.36148, 6.25486, 6.22526, 6.42692, 6.35485, 6.35029, 6.19105, 6.18567, 6.26859, 6.415, 6.23334, 6.18337, 6.21035, 6.14535, 6.09626, 6.10387, 6.28772, 6.43606, 6.29503, 6.335, 6.13464, 6.21503, 6.02829, 6.06095, 5.9935, 6.28273, 6.22023, 5.99847, 5.81393, 6.16265, 5.87946, 6.14445, 5.82485, 6.19248, 6.18157, 6.12584, 5.97074, 6.14877, 5.98325, 6.23524, 5.93942, 5.83892, 5.82229, 5.72934, 6.05496, 6.0434, 6.11051, 5.93954, 6.09171, 6.01241, 6.04004, 6.0322, 5.99651, 5.89061, 6.00653, 5.67122, 5.75784, 5.94696, 5.9005, 5.91468, 5.82189, 5.89471, 5.77842, 5.61622, 5.78054, 5.69253, 5.90048, 5.66647, 5.77352, 5.78152, 5.97131, 5.71328, 5.92696, 5.81669, 5.94504, 5.4175, 5.97213, 5.95642, 5.93165, 5.48932, 5.49949, 5.70719, 5.6873, 5.5725, 5.66702, 5.76913, 5.57229, 5.82826, 5.61559, 5.69173, 5.731, 5.73072, 5.62169, 5.71676, 5.78883, 5.80232, 5.67949, 5.77122, 5.47901, 5.79612, 5.73059, 5.53929, 5.69307, 5.7447, 5.6605, 5.44825, 5.66038, 5.60993, 5.60208, 5.50359, 5.67847, 5.72987, 5.52511, 5.65798, 5.63632, 5.4706, 5.64734, 5.55245, 5.58744, 5.44937, 5.20181, 5.63792, 5.72045, 5.87194, 5.56238, 5.74796, 5.79022, 5.38902, 5.44605, 5.54282, 5.55739, 5.49575, 5.64498, 5.33577, 5.45876, 5.42673, 5.5365, 5.42129, 5.62761, 5.71678, 5.48104, 5.60527, 5.5126, 5.25058, 5.49118, 5.43681, 5.48508, 5.28923, 5.46474, 5.45286, 5.6724, 5.35082, 5.46484, 5.40053, 5.54964, 5.16851, 5.10998, 5.5302, 5.59551, 5.43932, 5.53394, 5.2946, 5.37074, 5.47423, 5.2811, 5.46993, 5.28979, 5.57821, 5.48542, 5.37281, 5.45382, 5.27315, 5.53883, 5.2931, 5.25971, 5.35796, 5.33386, 5.5094, 5.38011, 5.51219, 5.30068, 5.34103, 5.49541, 5.54901, 5.50235, 5.43059, 5.39677, 5.52711, 5.19094, 5.45817, 5.34325, 5.56956, 5.41302, 5.43584, 5.37612, 5.25951, 5.25447, 5.49422, 5.5781, 5.35768, 5.3279, 5.19136, 5.4016, 5.39747, 5.20526, 5.61362, 5.29418, 5.39709, 5.44712, 5.30146, 5.34724, 5.36676, 5.28901, 5.361, 5.45905, 5.27649, 5.47318, 5.21725, 5.22023, 5.35122, 5.28396, 5.21834, 5.10071, 5.23602, 5.43096, 5.33142, 5.33017, 5.66246, 5.3004, 5.30692, 5.39386, 5.13475, 5.06957, 5.3365, 5.37793, 5.21244, 5.29887, 5.36995, 5.34675, 5.15473, 5.24757, 5.27856, 5.16172, 5.08869, 5.37568, 5.11393, 5.55309, 5.15317, 5.32295, 5.06795, 5.13265, 5.17242, 5.01042, 5.01637, 5.20515, 5.17193, 5.18392, 5.30507, 5.25233, 5.31569, 5.14154, 5.24356, 5.12106, 5.31092, 5.36465, 5.24729, 5.09639, 5.1804, 5.29568, 5.10464, 5.27827, 5.10619, 5.10892, 5.03572]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [12.43997, 12.4994, 12.67738, 12.01981, 11.40989, 9.15396, 6.91154, 7.19653, 6.10097, 4.66447, 4.20211, 2.8807, 2.37647, 2.34175, 2.05101, 2.19366, 2.12083, 1.89191, 2.18481, 2.06821, 2.11865, 2.16674, 2.00167, 2.19993, 1.94652, 2.02914, 1.87967, 1.849, 1.87625, 2.13926, 2.1644, 1.83737, 1.7865, 2.10617, 2.09168, 2.03916, 1.97963, 1.83822, 1.96495, 1.70803, 2.13244, 1.91303, 1.67031, 1.85063, 1.89388, 1.7393, 1.73696, 1.73834, 1.81384, 1.54681, 1.72306, 1.83162, 1.75476, 1.78654, 1.54973, 1.8348, 1.71396, 1.79871, 1.46752, 1.54685, 1.64797, 1.57656, 1.70218, 1.63082, 1.61792, 1.6742, 1.70617, 1.4063, 1.49439, 1.5398, 1.39435, 1.372, 1.63172, 1.45579, 1.3529, 1.50085, 1.31258, 1.33724, 1.14869, 1.28976, 1.19311, 1.38603, 1.20251, 1.31173, 1.10965, 1.18009, 1.42638, 1.54885, 1.1348, 1.01505, 1.06293, 1.23147, 0.95714, 0.89268, 0.94079, 1.27319, 1.18212, 1.01407, 1.03886, 1.50527, 1.02205, 1.09161, 0.91857, 1.10077, 0.94051, 1.19162, 0.99345, 0.96782, 1.0889, 0.98132, 1.29717, 0.8425, 1.11704, 0.95051, 1.15684, 0.97961, 0.94467, 1.05905, 0.93968, 1.14615, 0.96345, 0.97578, 1.19987, 0.96535, 1.25273, 1.46243, 1.21921, 0.99922, 1.14431, 1.34353, 1.06135, 1.14405, 1.10872, 1.1588, 0.94471, 1.01308, 0.94383, 0.99273, 0.97851, 0.89198, 1.09779, 1.31177, 1.05508, 0.91714, 1.0117, 1.28832, 1.09784, 1.19667, 0.92098, 0.98378, 1.03891, 1.07858, 1.29929, 0.94354, 1.06388, 1.50705, 1.0007, 1.35362, 1.28287, 0.84574, 1.11813, 1.1825, 1.04876, 1.12893, 1.16116, 1.12585, 1.11897, 1.15162, 1.30322, 1.20265, 1.018, 0.99879, 0.90328, 1.21092, 1.0701, 1.06218, 1.10403, 1.0926, 1.05063, 1.07573, 1.20003, 1.25848, 1.34649, 1.12066, 1.50822, 1.14324, 1.4787, 1.1305, 1.14505, 1.16533, 1.14287, 1.24641, 1.38816, 1.42518, 1.1866, 1.45857, 1.17698, 1.2263, 1.01505, 1.21325, 1.36272, 1.305, 1.19874, 1.18217, 1.01807, 1.24602, 1.46217, 1.22746, 1.20492, 1.3465, 1.12878, 1.16877, 1.06974, 1.08696, 1.6092, 1.25397, 1.20201, 1.08861, 1.34872, 1.27688, 1.5104, 1.30437, 1.05297, 1.3032, 1.2672, 1.36045, 1.15533, 1.08165, 1.20493, 1.17126, 1.18099, 1.25764, 1.52555, 1.33265, 1.17044, 1.32121, 1.21081, 1.39328, 1.50488, 1.28381, 1.24675, 1.23603, 1.3193, 1.29405, 1.23259, 1.07163, 1.1052, 1.24045, 1.37927, 1.50839, 1.32285, 1.38782, 1.13484, 1.21127, 2.00278, 1.36691, 1.32213, 1.37434, 1.00254, 1.08214, 1.17335, 1.41525, 1.25392, 1.43316, 1.39572, 1.31067, 1.2846, 1.09515, 1.18724, 1.20128, 1.30643, 1.23357, 1.11402, 1.17568, 1.29277, 1.22678, 1.1362, 1.18826, 1.25873, 1.2814, 1.22295, 1.02105, 1.29626, 1.3106, 1.38573, 1.28368, 1.04758, 1.13079, 1.06747, 1.51913, 1.45844, 1.11656, 1.1972, 1.22395, 1.4347, 1.41031, 1.11466, 1.5639, 1.36293, 1.24572, 1.4447, 1.25296, 1.14388, 1.12495, 1.31276, 1.35398, 1.2105, 1.44264, 1.16726, 1.19041, 1.35889, 1.20903, 1.15845, 1.12041, 1.06639, 1.2833, 1.21736, 1.18244, 1.41925, 1.21164, 1.17543, 1.27955, 1.27399, 1.23019, 1.33022, 1.24584, 1.546, 1.32952, 1.1706, 1.31643, 1.32431, 1.26323, 1.13097, 1.34316, 1.10348, 1.33974, 1.18037, 1.18919, 1.42354, 1.37144, 1.33382, 1.39443, 1.37347, 1.18285, 1.1776, 1.31269, 1.10901, 1.33507, 1.39353, 1.28869, 1.32106, 1.36384, 1.307, 1.2118, 1.20055, 1.076, 1.20907, 1.28103, 1.2481, 1.49609, 1.25261, 1.22933, 1.23135, 1.40382, 1.47949, 1.50263, 1.27893, 1.27615, 1.34666, 1.30354, 1.1997, 1.51644, 1.42165, 1.35804, 1.19426, 1.23401, 1.36501, 1.05637, 1.11768, 1.22237, 1.39349, 1.3636, 1.33587, 1.44787, 1.23775, 1.25341, 1.15189, 1.07392, 1.29463, 1.16475, 1.13311, 1.32307, 1.04489, 1.17108, 1.24996, 1.21235, 1.90656, 1.20192, 1.24416, 1.32035]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [12.43997, 12.4994, 12.67738, 12.01981, 11.40989, 9.15396, 6.91154, 7.19653, 6.10097, 4.66447, 4.20211, 2.8807, 2.37647, 2.34175, 2.05101, 2.19366, 2.12083, 1.89191, 2.18481, 2.06821, 2.11865, 2.16674, 2.00167, 2.19993, 1.94652, 2.02914, 1.87967, 1.849, 1.87625, 2.13926, 2.1644, 1.83737, 1.7865, 2.10617, 2.09168, 2.03916, 1.97963, 1.83822, 1.96495, 1.70803, 2.13244, 1.91303, 1.67031, 1.85063, 1.89388, 1.7393, 1.73696, 1.73834, 1.81384, 1.54681, 1.72306, 1.83162, 1.75476, 1.78654, 1.54973, 1.8348, 1.71396, 1.79871, 1.46752, 1.54685, 1.64797, 1.57656, 1.70218, 1.63082, 1.61792, 1.6742, 1.70617, 1.4063, 1.49439, 1.5398, 1.39435, 1.372, 1.63172, 1.45579, 1.3529, 1.50085, 1.31258, 1.33724, 1.14869, 1.28976, 1.19311, 1.38603, 1.20251, 1.31173, 1.10965, 1.18009, 1.42638, 1.54885, 1.1348, 1.01505, 1.06293, 1.23147, 0.95714, 0.89268, 0.94079, 1.27319, 1.18212, 1.01407, 1.03886, 1.50527, 1.02205, 1.09161, 0.91857, 1.10077, 0.94051, 1.19162, 0.99345, 0.96782, 1.0889, 0.98132, 1.29717, 0.8425, 1.11704, 0.95051, 1.15684, 0.97961, 0.94467, 1.05905, 0.93968, 1.14615, 0.96345, 0.97578, 1.19987, 0.96535, 1.25273, 1.46243, 1.21921, 0.99922, 1.14431, 1.34353, 1.06135, 1.14405, 1.10872, 1.1588, 0.94471, 1.01308, 0.94383, 0.99273, 0.97851, 0.89198, 1.09779, 1.31177, 1.05508, 0.91714, 1.0117, 1.28832, 1.09784, 1.19667, 0.92098, 0.98378, 1.03891, 1.07858, 1.29929, 0.94354, 1.06388, 1.50705, 1.0007, 1.35362, 1.28287, 0.84574, 1.11813, 1.1825, 1.04876, 1.12893, 1.16116, 1.12585, 1.11897, 1.15162, 1.30322, 1.20265, 1.018, 0.99879, 0.90328, 1.21092, 1.0701, 1.06218, 1.10403, 1.0926, 1.05063, 1.07573, 1.20003, 1.25848, 1.34649, 1.12066, 1.50822, 1.14324, 1.4787, 1.1305, 1.14505, 1.16533, 1.14287, 1.24641, 1.38816, 1.42518, 1.1866, 1.45857, 1.17698, 1.2263, 1.01505, 1.21325, 1.36272, 1.305, 1.19874, 1.18217, 1.01807, 1.24602, 1.46217, 1.22746, 1.20492, 1.3465, 1.12878, 1.16877, 1.06974, 1.08696, 1.6092, 1.25397, 1.20201, 1.08861, 1.34872, 1.27688, 1.5104, 1.30437, 1.05297, 1.3032, 1.2672, 1.36045, 1.15533, 1.08165, 1.20493, 1.17126, 1.18099, 1.25764, 1.52555, 1.33265, 1.17044, 1.32121, 1.21081, 1.39328, 1.50488, 1.28381, 1.24675, 1.23603, 1.3193, 1.29405, 1.23259, 1.07163, 1.1052, 1.24045, 1.37927, 1.50839, 1.32285, 1.38782, 1.13484, 1.21127, 2.00278, 1.36691, 1.32213, 1.37434, 1.00254, 1.08214, 1.17335, 1.41525, 1.25392, 1.43316, 1.39572, 1.31067, 1.2846, 1.09515, 1.18724, 1.20128, 1.30643, 1.23357, 1.11402, 1.17568, 1.29277, 1.22678, 1.1362, 1.18826, 1.25873, 1.2814, 1.22295, 1.02105, 1.29626, 1.3106, 1.38573, 1.28368, 1.04758, 1.13079, 1.06747, 1.51913, 1.45844, 1.11656, 1.1972, 1.22395, 1.4347, 1.41031, 1.11466, 1.5639, 1.36293, 1.24572, 1.4447, 1.25296, 1.14388, 1.12495, 1.31276, 1.35398, 1.2105, 1.44264, 1.16726, 1.19041, 1.35889, 1.20903, 1.15845, 1.12041, 1.06639, 1.2833, 1.21736, 1.18244, 1.41925, 1.21164, 1.17543, 1.27955, 1.27399, 1.23019, 1.33022, 1.24584, 1.546, 1.32952, 1.1706, 1.31643, 1.32431, 1.26323, 1.13097, 1.34316, 1.10348, 1.33974, 1.18037, 1.18919, 1.42354, 1.37144, 1.33382, 1.39443, 1.37347, 1.18285, 1.1776, 1.31269, 1.10901, 1.33507, 1.39353, 1.28869, 1.32106, 1.36384, 1.307, 1.2118, 1.20055, 1.076, 1.20907, 1.28103, 1.2481, 1.49609, 1.25261, 1.22933, 1.23135, 1.40382, 1.47949, 1.50263, 1.27893, 1.27615, 1.34666, 1.30354, 1.1997, 1.51644, 1.42165, 1.35804, 1.19426, 1.23401, 1.36501, 1.05637, 1.11768, 1.22237, 1.39349, 1.3636, 1.33587, 1.44787, 1.23775, 1.25341, 1.15189, 1.07392, 1.29463, 1.16475, 1.13311, 1.32307, 1.04489, 1.17108, 1.24996, 1.21235, 1.90656, 1.20192, 1.24416, 1.32035]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [80.0, 89.0, 102.0, 88.0, 78.0, 115.0, 125.0, 114.0, 129.0, 106.0, 125.0, 179.0, 156.0, 184.0, 179.0, 191.0, 171.0, 216.0, 169.0, 200.0, 171.0, 184.0, 206.0, 173.0, 221.0, 181.0, 188.0, 209.0, 187.0, 188.0, 167.0, 165.0, 180.0, 204.0, 152.0, 155.0, 170.0, 179.0, 177.0, 197.0, 184.0, 162.0, 194.0, 184.0, 171.0, 206.0, 198.0, 200.0, 187.0, 238.0, 208.0, 173.0, 201.0, 145.0, 199.0, 194.0, 185.0, 173.0, 266.0, 238.0, 190.0, 195.0, 182.0, 188.0, 199.0, 262.0, 210.0, 233.0, 216.0, 199.0, 257.0, 213.0, 220.0, 243.0, 218.0, 215.0, 229.0, 219.0, 289.0, 212.0, 280.0, 229.0, 196.0, 274.0, 237.0, 246.0, 170.0, 203.0, 205.0, 236.0, 201.0, 203.0, 256.0, 220.0, 191.0, 173.0, 214.0, 225.0, 183.0, 151.0, 195.0, 174.0, 218.0, 189.0, 159.0, 151.0, 154.0, 154.0, 130.0, 202.0, 162.0, 186.0, 166.0, 187.0, 136.0, 145.0, 168.0, 100.0, 161.0, 124.0, 138.0, 163.0, 108.0, 167.0, 129.0, 131.0, 141.0, 148.0, 128.0, 124.0, 137.0, 168.0, 133.0, 114.0, 139.0, 123.0, 161.0, 139.0, 133.0, 152.0, 122.0, 111.0, 135.0, 155.0, 158.0, 101.0, 134.0, 164.0, 136.0, 163.0, 110.0, 153.0, 116.0, 132.0, 120.0, 115.0, 108.0, 85.0, 97.0, 169.0, 112.0, 115.0, 134.0, 105.0, 114.0, 156.0, 115.0, 103.0, 125.0, 113.0, 121.0, 138.0, 114.0, 130.0, 122.0, 118.0, 88.0, 106.0, 113.0, 121.0, 134.0, 131.0, 118.0, 130.0, 93.0, 111.0, 114.0, 111.0, 106.0, 95.0, 105.0, 107.0, 107.0, 87.0, 112.0, 90.0, 116.0, 104.0, 135.0, 140.0, 102.0, 104.0, 142.0, 144.0, 121.0, 87.0, 99.0, 136.0, 115.0, 105.0, 126.0, 112.0, 126.0, 125.0, 115.0, 116.0, 121.0, 145.0, 109.0, 111.0, 103.0, 112.0, 129.0, 115.0, 130.0, 97.0, 119.0, 103.0, 116.0, 135.0, 109.0, 115.0, 109.0, 113.0, 119.0, 116.0, 105.0, 107.0, 105.0, 109.0, 113.0, 115.0, 101.0, 114.0, 109.0, 123.0, 111.0, 117.0, 106.0, 92.0, 103.0, 118.0, 116.0, 130.0, 99.0, 107.0, 121.0, 96.0, 124.0, 112.0, 134.0, 104.0, 115.0, 104.0, 113.0, 107.0, 119.0, 124.0, 116.0, 115.0, 123.0, 139.0, 117.0, 118.0, 110.0, 112.0, 124.0, 112.0, 104.0, 98.0, 108.0, 134.0, 108.0, 126.0, 123.0, 118.0, 120.0, 122.0, 141.0, 105.0, 81.0, 122.0, 131.0, 123.0, 122.0, 101.0, 129.0, 88.0, 131.0, 124.0, 110.0, 124.0, 130.0, 141.0, 109.0, 107.0, 95.0, 104.0, 136.0, 123.0, 121.0, 123.0, 111.0, 117.0, 142.0, 120.0, 111.0, 108.0, 86.0, 121.0, 115.0, 111.0, 125.0, 128.0, 93.0, 126.0, 116.0, 124.0, 94.0, 107.0, 107.0, 128.0, 106.0, 110.0, 128.0, 104.0, 105.0, 114.0, 118.0, 117.0, 99.0, 123.0, 108.0, 107.0, 126.0, 119.0, 121.0, 121.0, 107.0, 116.0, 116.0, 116.0, 126.0, 145.0, 132.0, 133.0, 125.0, 100.0, 98.0, 129.0, 118.0, 121.0, 105.0, 107.0, 95.0, 113.0, 106.0, 108.0, 94.0, 121.0, 139.0, 118.0, 101.0, 98.0, 111.0, 117.0, 112.0, 129.0, 113.0, 119.0, 103.0, 123.0, 124.0, 107.0, 121.0, 117.0, 126.0, 123.0, 103.0, 113.0, 131.0, 117.0, 128.0, 123.0, 103.0, 149.0, 113.0, 101.0, 122.0, 110.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [80.0, 89.0, 102.0, 88.0, 78.0, 115.0, 125.0, 114.0, 129.0, 106.0, 125.0, 179.0, 156.0, 184.0, 179.0, 191.0, 171.0, 216.0, 169.0, 200.0, 171.0, 184.0, 206.0, 173.0, 221.0, 181.0, 188.0, 209.0, 187.0, 188.0, 167.0, 165.0, 180.0, 204.0, 152.0, 155.0, 170.0, 179.0, 177.0, 197.0, 184.0, 162.0, 194.0, 184.0, 171.0, 206.0, 198.0, 200.0, 187.0, 238.0, 208.0, 173.0, 201.0, 145.0, 199.0, 194.0, 185.0, 173.0, 266.0, 238.0, 190.0, 195.0, 182.0, 188.0, 199.0, 262.0, 210.0, 233.0, 216.0, 199.0, 257.0, 213.0, 220.0, 243.0, 218.0, 215.0, 229.0, 219.0, 289.0, 212.0, 280.0, 229.0, 196.0, 274.0, 237.0, 246.0, 170.0, 203.0, 205.0, 236.0, 201.0, 203.0, 256.0, 220.0, 191.0, 173.0, 214.0, 225.0, 183.0, 151.0, 195.0, 174.0, 218.0, 189.0, 159.0, 151.0, 154.0, 154.0, 130.0, 202.0, 162.0, 186.0, 166.0, 187.0, 136.0, 145.0, 168.0, 100.0, 161.0, 124.0, 138.0, 163.0, 108.0, 167.0, 129.0, 131.0, 141.0, 148.0, 128.0, 124.0, 137.0, 168.0, 133.0, 114.0, 139.0, 123.0, 161.0, 139.0, 133.0, 152.0, 122.0, 111.0, 135.0, 155.0, 158.0, 101.0, 134.0, 164.0, 136.0, 163.0, 110.0, 153.0, 116.0, 132.0, 120.0, 115.0, 108.0, 85.0, 97.0, 169.0, 112.0, 115.0, 134.0, 105.0, 114.0, 156.0, 115.0, 103.0, 125.0, 113.0, 121.0, 138.0, 114.0, 130.0, 122.0, 118.0, 88.0, 106.0, 113.0, 121.0, 134.0, 131.0, 118.0, 130.0, 93.0, 111.0, 114.0, 111.0, 106.0, 95.0, 105.0, 107.0, 107.0, 87.0, 112.0, 90.0, 116.0, 104.0, 135.0, 140.0, 102.0, 104.0, 142.0, 144.0, 121.0, 87.0, 99.0, 136.0, 115.0, 105.0, 126.0, 112.0, 126.0, 125.0, 115.0, 116.0, 121.0, 145.0, 109.0, 111.0, 103.0, 112.0, 129.0, 115.0, 130.0, 97.0, 119.0, 103.0, 116.0, 135.0, 109.0, 115.0, 109.0, 113.0, 119.0, 116.0, 105.0, 107.0, 105.0, 109.0, 113.0, 115.0, 101.0, 114.0, 109.0, 123.0, 111.0, 117.0, 106.0, 92.0, 103.0, 118.0, 116.0, 130.0, 99.0, 107.0, 121.0, 96.0, 124.0, 112.0, 134.0, 104.0, 115.0, 104.0, 113.0, 107.0, 119.0, 124.0, 116.0, 115.0, 123.0, 139.0, 117.0, 118.0, 110.0, 112.0, 124.0, 112.0, 104.0, 98.0, 108.0, 134.0, 108.0, 126.0, 123.0, 118.0, 120.0, 122.0, 141.0, 105.0, 81.0, 122.0, 131.0, 123.0, 122.0, 101.0, 129.0, 88.0, 131.0, 124.0, 110.0, 124.0, 130.0, 141.0, 109.0, 107.0, 95.0, 104.0, 136.0, 123.0, 121.0, 123.0, 111.0, 117.0, 142.0, 120.0, 111.0, 108.0, 86.0, 121.0, 115.0, 111.0, 125.0, 128.0, 93.0, 126.0, 116.0, 124.0, 94.0, 107.0, 107.0, 128.0, 106.0, 110.0, 128.0, 104.0, 105.0, 114.0, 118.0, 117.0, 99.0, 123.0, 108.0, 107.0, 126.0, 119.0, 121.0, 121.0, 107.0, 116.0, 116.0, 116.0, 126.0, 145.0, 132.0, 133.0, 125.0, 100.0, 98.0, 129.0, 118.0, 121.0, 105.0, 107.0, 95.0, 113.0, 106.0, 108.0, 94.0, 121.0, 139.0, 118.0, 101.0, 98.0, 111.0, 117.0, 112.0, 129.0, 113.0, 119.0, 103.0, 123.0, 124.0, 107.0, 121.0, 117.0, 126.0, 123.0, 103.0, 113.0, 131.0, 117.0, 128.0, 123.0, 103.0, 149.0, 113.0, 101.0, 122.0, 110.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95625, 179.95625, 179.95625, 179.95625, 179.95625, 179.95624, 179.95622, 179.95612, 179.95593, 179.95575, 179.95451, 179.95384, 179.95331, 179.95131, 179.95029, 179.94963, 179.94899, 179.94896, 179.94923, 179.94928, 179.94922, 179.94897, 179.94885, 179.9491, 179.94991, 179.951, 179.95213, 179.95309, 179.95415, 179.95551, 179.9574, 179.95952, 179.96179, 179.96399, 179.96649, 179.96965, 179.97318, 179.97679, 179.98051, 179.98468, 179.98955, 179.99477, 180.00044, 180.00658, 180.01337, 180.02075, 180.02858, 180.03702, 180.04625, 180.05624, 180.06699, 180.0782, 180.09018, 180.10277, 180.11606, 180.12999, 180.14421, 180.159, 180.17467, 180.19148, 180.20897, 180.22713, 180.24684, 180.26782, 180.2896, 180.31204, 180.33545, 180.35973, 180.38542, 180.41144, 180.43797, 180.46524, 180.4928, 180.52104, 180.54993, 180.57939, 180.60922, 180.63998, 180.67151, 180.70398, 180.73651, 180.76875, 180.80157, 180.83536, 180.86948, 180.90508, 180.9411, 180.97647, 181.01176, 181.04828, 181.08588, 181.12448, 181.16327, 181.20253, 181.24295, 181.28366, 181.32249, 181.35963, 181.39644, 181.43352, 181.47067, 181.50752, 181.54518, 181.58394, 181.62318, 181.66335, 181.7032, 181.74304, 181.78291, 181.82195, 181.86037, 181.89832, 181.93773, 181.97792, 182.01897, 182.05927, 182.09976, 182.14062, 182.18091, 182.22133, 182.26169, 182.30261, 182.34355, 182.38451, 182.4248, 182.46426, 182.50208, 182.53731, 182.57451, 182.61168, 182.64999, 182.68562, 182.72139, 182.75731, 182.79347, 182.83156, 182.87192, 182.91328, 182.95439, 182.99614, 183.03891, 183.07968, 183.12061, 183.16183, 183.20284, 183.24399, 183.28496, 183.325, 183.3662, 183.40788, 183.45087, 183.49307, 183.53464, 183.57661, 183.61989, 183.66231, 183.70183, 183.7419, 183.78094, 183.81953, 183.86018, 183.90375, 183.94774, 183.9931, 184.03831, 184.08267, 184.12688, 184.16986, 184.21062, 184.25189, 184.29411, 184.3373, 184.38132, 184.42554, 184.46965, 184.51401, 184.55882, 184.60381, 184.64806, 184.69025, 184.73256, 184.7748, 184.817, 184.86073, 184.90417, 184.94685, 184.98766, 185.02675, 185.06696, 185.10852, 185.15274, 185.19722, 185.24055, 185.28352, 185.32553, 185.36723, 185.40932, 185.45212, 185.49559, 185.54068, 185.58374, 185.62703, 185.6687, 185.71231, 185.75662, 185.80209, 185.84537, 185.88788, 185.93077, 185.97299, 186.01599, 186.05911, 186.10475, 186.15176, 186.19826, 186.24303, 186.28674, 186.33194, 186.377, 186.42128, 186.46397, 186.50703, 186.55083, 186.59554, 186.63943, 186.68254, 186.72632, 186.77109, 186.81587, 186.86107, 186.90485, 186.94669, 186.9883, 187.03162, 187.07474, 187.11856, 187.16187, 187.20621, 187.25069, 187.29416, 187.33778, 187.38162, 187.42618, 187.47089, 187.51416, 187.56001, 187.60674, 187.6539, 187.70016, 187.74496, 187.7905, 187.83824, 187.88522, 187.93312, 187.98019, 188.02357, 188.06801, 188.11484, 188.1615, 188.21011, 188.26111, 188.31125, 188.35876, 188.4053, 188.45084, 188.49641, 188.54265, 188.58983, 188.64067, 188.69183, 188.74222, 188.79266, 188.84273, 188.89304, 188.94508, 188.99475, 189.04398, 189.09485, 189.14598, 189.1965, 189.24777, 189.29964, 189.35378, 189.40587, 189.45831, 189.50987, 189.56148, 189.61368, 189.66797, 189.71982, 189.77005, 189.81833, 189.86722, 189.91873, 189.97101, 190.02145, 190.07199, 190.12384, 190.17366, 190.22346, 190.27402, 190.3253, 190.37793, 190.43097, 190.48424, 190.53532, 190.58551, 190.63808, 190.69084, 190.74536, 190.79968, 190.85349, 190.90894, 190.96626, 191.02402, 191.08208, 191.13948, 191.19746, 191.25615, 191.31114, 191.36597, 191.4203, 191.47542, 191.53027, 191.58527, 191.63684, 191.68701, 191.73514, 191.78677, 191.83801, 191.8905, 191.94266, 191.99596, 192.05061, 192.1071, 192.16386, 192.21751, 192.27289, 192.32852, 192.37949, 192.43187, 192.48483, 192.53804, 192.59248, 192.64667, 192.70181, 192.75798, 192.81502, 192.87016, 192.92496, 192.98015, 193.03481, 193.09019, 193.14693, 193.20465, 193.26526, 193.32504, 193.38451, 193.44281, 193.49977, 193.55804, 193.61533, 193.67177, 193.72891, 193.78667, 193.84259, 193.89799, 193.95425, 194.01086, 194.06876, 194.12726, 194.18596, 194.24385, 194.30168, 194.35782, 194.41516, 194.47411, 194.53342, 194.59587, 194.65793, 194.71797, 194.77441, 194.83284, 194.88989, 194.94766, 195.00539, 195.06413, 195.12605, 195.19096, 195.25722, 195.32449, 195.39157, 195.45724, 195.52281, 195.58981, 195.65671, 195.7216, 195.78194, 195.84415, 195.90858]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95625, 179.95625, 179.95625, 179.95625, 179.95625, 179.95624, 179.95622, 179.95612, 179.95593, 179.95575, 179.95451, 179.95384, 179.95331, 179.95131, 179.95029, 179.94963, 179.94899, 179.94896, 179.94923, 179.94928, 179.94922, 179.94897, 179.94885, 179.9491, 179.94991, 179.951, 179.95213, 179.95309, 179.95415, 179.95551, 179.9574, 179.95952, 179.96179, 179.96399, 179.96649, 179.96965, 179.97318, 179.97679, 179.98051, 179.98468, 179.98955, 179.99477, 180.00044, 180.00658, 180.01337, 180.02075, 180.02858, 180.03702, 180.04625, 180.05624, 180.06699, 180.0782, 180.09018, 180.10277, 180.11606, 180.12999, 180.14421, 180.159, 180.17467, 180.19148, 180.20897, 180.22713, 180.24684, 180.26782, 180.2896, 180.31204, 180.33545, 180.35973, 180.38542, 180.41144, 180.43797, 180.46524, 180.4928, 180.52104, 180.54993, 180.57939, 180.60922, 180.63998, 180.67151, 180.70398, 180.73651, 180.76875, 180.80157, 180.83536, 180.86948, 180.90508, 180.9411, 180.97647, 181.01176, 181.04828, 181.08588, 181.12448, 181.16327, 181.20253, 181.24295, 181.28366, 181.32249, 181.35963, 181.39644, 181.43352, 181.47067, 181.50752, 181.54518, 181.58394, 181.62318, 181.66335, 181.7032, 181.74304, 181.78291, 181.82195, 181.86037, 181.89832, 181.93773, 181.97792, 182.01897, 182.05927, 182.09976, 182.14062, 182.18091, 182.22133, 182.26169, 182.30261, 182.34355, 182.38451, 182.4248, 182.46426, 182.50208, 182.53731, 182.57451, 182.61168, 182.64999, 182.68562, 182.72139, 182.75731, 182.79347, 182.83156, 182.87192, 182.91328, 182.95439, 182.99614, 183.03891, 183.07968, 183.12061, 183.16183, 183.20284, 183.24399, 183.28496, 183.325, 183.3662, 183.40788, 183.45087, 183.49307, 183.53464, 183.57661, 183.61989, 183.66231, 183.70183, 183.7419, 183.78094, 183.81953, 183.86018, 183.90375, 183.94774, 183.9931, 184.03831, 184.08267, 184.12688, 184.16986, 184.21062, 184.25189, 184.29411, 184.3373, 184.38132, 184.42554, 184.46965, 184.51401, 184.55882, 184.60381, 184.64806, 184.69025, 184.73256, 184.7748, 184.817, 184.86073, 184.90417, 184.94685, 184.98766, 185.02675, 185.06696, 185.10852, 185.15274, 185.19722, 185.24055, 185.28352, 185.32553, 185.36723, 185.40932, 185.45212, 185.49559, 185.54068, 185.58374, 185.62703, 185.6687, 185.71231, 185.75662, 185.80209, 185.84537, 185.88788, 185.93077, 185.97299, 186.01599, 186.05911, 186.10475, 186.15176, 186.19826, 186.24303, 186.28674, 186.33194, 186.377, 186.42128, 186.46397, 186.50703, 186.55083, 186.59554, 186.63943, 186.68254, 186.72632, 186.77109, 186.81587, 186.86107, 186.90485, 186.94669, 186.9883, 187.03162, 187.07474, 187.11856, 187.16187, 187.20621, 187.25069, 187.29416, 187.33778, 187.38162, 187.42618, 187.47089, 187.51416, 187.56001, 187.60674, 187.6539, 187.70016, 187.74496, 187.7905, 187.83824, 187.88522, 187.93312, 187.98019, 188.02357, 188.06801, 188.11484, 188.1615, 188.21011, 188.26111, 188.31125, 188.35876, 188.4053, 188.45084, 188.49641, 188.54265, 188.58983, 188.64067, 188.69183, 188.74222, 188.79266, 188.84273, 188.89304, 188.94508, 188.99475, 189.04398, 189.09485, 189.14598, 189.1965, 189.24777, 189.29964, 189.35378, 189.40587, 189.45831, 189.50987, 189.56148, 189.61368, 189.66797, 189.71982, 189.77005, 189.81833, 189.86722, 189.91873, 189.97101, 190.02145, 190.07199, 190.12384, 190.17366, 190.22346, 190.27402, 190.3253, 190.37793, 190.43097, 190.48424, 190.53532, 190.58551, 190.63808, 190.69084, 190.74536, 190.79968, 190.85349, 190.90894, 190.96626, 191.02402, 191.08208, 191.13948, 191.19746, 191.25615, 191.31114, 191.36597, 191.4203, 191.47542, 191.53027, 191.58527, 191.63684, 191.68701, 191.73514, 191.78677, 191.83801, 191.8905, 191.94266, 191.99596, 192.05061, 192.1071, 192.16386, 192.21751, 192.27289, 192.32852, 192.37949, 192.43187, 192.48483, 192.53804, 192.59248, 192.64667, 192.70181, 192.75798, 192.81502, 192.87016, 192.92496, 192.98015, 193.03481, 193.09019, 193.14693, 193.20465, 193.26526, 193.32504, 193.38451, 193.44281, 193.49977, 193.55804, 193.61533, 193.67177, 193.72891, 193.78667, 193.84259, 193.89799, 193.95425, 194.01086, 194.06876, 194.12726, 194.18596, 194.24385, 194.30168, 194.35782, 194.41516, 194.47411, 194.53342, 194.59587, 194.65793, 194.71797, 194.77441, 194.83284, 194.88989, 194.94766, 195.00539, 195.06413, 195.12605, 195.19096, 195.25722, 195.32449, 195.39157, 195.45724, 195.52281, 195.58981, 195.65671, 195.7216, 195.78194, 195.84415, 195.90858]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [18.92793, 0.51136, 0.50959, 0.5023, 0.50706, 0.49889, 0.49918, 0.50787, 0.50805, 0.50023, 0.51244, 0.49782, 0.5011, 0.49829, 0.50242, 0.49765, 0.50512, 0.50815, 0.51211, 0.49886, 0.50327, 0.50436, 0.50354, 0.4972, 0.49868, 0.50277, 0.49981, 0.50008, 0.50203, 0.49718, 0.60026, 0.49876, 0.49477, 0.5046, 0.51537, 0.5196, 0.49706, 0.49993, 0.49908, 0.49804, 0.4994, 0.49794, 0.50015, 0.49859, 0.49669, 0.49649, 0.59124, 0.49837, 0.50138, 0.49717, 0.49966, 0.50461, 0.4977, 0.49673, 0.5025, 0.49998, 0.49865, 0.50151, 0.50846, 0.51111, 0.50552, 0.50429, 0.50589, 0.50627, 0.50795, 0.505, 0.50478, 0.50608, 0.5063, 0.50392, 0.50528, 0.50464, 0.50852, 0.50732, 0.50975, 0.70338, 0.50322, 0.50607, 0.5008, 0.51264, 0.50202, 0.51117, 0.50466, 0.50856, 0.50482, 0.5101, 0.50604, 0.50708, 0.50371, 0.50732, 0.50754, 0.50725, 0.50576, 0.50944, 0.50954, 0.50758, 0.50654, 0.5929, 0.50552, 0.50521, 0.50353, 0.50768, 0.50269, 0.50818, 0.50339, 0.50584, 0.50369, 0.50801, 0.50311, 0.50501, 0.50259, 0.50478, 0.50477, 0.50612, 0.50304, 0.5048, 0.50419, 0.50917, 0.50259, 0.59305, 0.71675, 0.50782, 0.50595, 0.50366, 0.50416, 0.5131, 0.50874, 0.50202, 0.5075, 0.50344, 0.50969, 0.50236, 0.50738, 0.5042, 0.50968, 0.50453, 0.50797, 0.50316, 0.50801, 0.50385, 0.51048, 0.50461, 0.60109, 0.50835, 0.50599, 0.50503, 0.50405, 0.50686, 0.50365, 0.50633, 0.51394, 0.507, 0.50416, 0.5072, 0.50187, 0.50987, 0.50554, 0.50964, 0.49997, 0.5086, 0.50287, 0.50901, 0.51253, 0.51268, 0.59174, 0.63218, 0.50352, 0.50458, 0.50663, 0.50624, 0.50529, 0.50834, 0.50628, 0.50536, 0.50697, 0.50514, 0.5058, 0.5064, 0.51003, 0.50482, 0.50622, 0.50306, 0.50955, 0.50288, 0.51052, 0.50915, 0.50819, 0.50518, 0.50395, 0.50908, 0.50261, 0.5111, 0.59558, 0.50726, 0.50659, 0.50692, 0.50765, 0.50516, 0.51034, 0.50537, 0.49111, 0.50535, 0.50465, 0.50275, 0.50558, 0.5014, 0.5079, 0.5078, 0.50568, 0.5069, 0.50614, 0.50631, 0.5066, 0.50398, 0.50618, 0.50721, 0.51171, 0.50602, 0.50818, 0.50511, 0.51286, 0.50398, 0.50849, 0.50801, 0.50817, 0.50985, 0.50547, 0.50729, 0.50608, 0.59229, 0.50801, 0.50242, 0.51408, 0.50883, 0.5042, 0.508, 0.51821, 0.50964, 0.50309, 0.51214, 0.59459, 0.51016, 0.50757, 0.51259, 0.50854, 0.50258, 0.50468, 0.50579, 0.50859, 0.50372, 0.50798, 0.50757, 0.51184, 0.50914, 0.50776, 0.50432, 0.50917, 0.50287, 0.50616, 0.50167, 0.5065, 0.50145, 0.51091, 0.50163, 0.51326, 0.50092, 0.50601, 0.50447, 0.50502, 0.50274, 0.50572, 0.50976, 0.5047, 0.50868, 0.50316, 0.52048, 0.50699, 0.61568, 0.50722, 0.5088, 0.50773, 0.50579, 0.50532, 0.50689, 0.50615, 0.50762, 0.5023, 0.50258, 0.50262, 0.51065, 0.50567, 0.50633, 0.50361, 0.50893, 0.50511, 0.50936, 0.59793, 0.60202, 0.51102, 0.50683, 0.50341, 0.50975, 0.50313, 0.51068, 0.50494, 0.5094, 0.50552, 0.5077, 0.50574, 0.50655, 0.51164, 0.50641, 0.50789, 0.50671, 0.61258, 0.50815, 0.50767, 0.50856, 0.51335, 0.5105, 0.50233, 0.50903, 0.50975, 0.50328, 0.50987, 0.50357, 0.50951, 0.50423, 0.50818, 0.50563, 0.50771, 0.50968, 0.50443, 0.50847, 0.50717, 0.50752, 0.50453, 0.50914, 0.50657, 0.50601, 0.51204, 0.50439, 0.59526, 0.50772, 0.50461, 0.51966, 0.50388, 0.50764, 0.50335, 0.51566, 0.50622, 0.50664, 0.50857, 0.51175, 0.50837, 0.50352, 0.50963, 0.50442, 0.50747, 0.50672, 0.50844, 0.50629, 0.50717, 0.5071, 0.50387, 0.5066, 0.50594, 0.50388, 0.50981, 0.50538, 0.5055, 0.50641, 0.50813, 0.50422, 0.50345, 0.50462, 0.50731, 0.50278, 0.50356, 0.50701, 0.5066, 0.5073, 0.51, 0.50394, 0.50873, 0.50751, 0.50848, 0.59448, 0.50862, 0.5117, 0.50484, 0.51229, 0.50735, 0.50392, 0.50744, 0.50609, 0.50765, 0.51917, 0.51153, 0.50229]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.68727]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.68727]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [295.08755]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [295.08755]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/model_config.yaml
index c43821c3a8..4349bc01a3 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_nondet_tp1_pp1_fp8_no_model_parallel/model_config.yaml
@@ -42,7 +42,7 @@ MODEL_ARGS:
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/golden_values.json
new file mode 100644
index 0000000000..fdeaa49aa1
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [17.4566, 0.37175, 0.37134, 0.37017, 0.37156, 0.37759, 0.37765, 0.37162, 0.3761, 0.37226, 0.53616, 0.37589, 0.37516, 0.37683, 0.37327, 0.37614, 0.37342, 0.3739, 0.37649, 0.37491, 0.38081, 0.37232, 0.37401, 0.37224, 0.37132, 0.38167, 0.37456, 0.37215, 0.36647, 0.37435, 0.38453, 0.36353, 0.36605, 0.36205, 0.36329, 0.36758, 0.36245, 0.36564, 0.3674, 0.38594, 0.36767, 0.36685, 0.36727, 0.36428, 0.3664, 0.36716, 0.36619, 0.36593, 0.36805, 0.36393, 0.3666, 0.36486, 0.36817, 0.36273, 0.36485, 0.36634, 0.36443, 0.3672, 0.36462, 0.36335, 0.35994, 0.36774, 0.36167, 0.36089, 0.36216, 0.36236, 0.36412, 0.36497, 0.3673, 0.36303, 0.36566, 0.36239, 0.36323, 0.36008, 0.46258, 0.36181, 0.3621, 0.36509, 0.36772, 0.36417, 0.36489, 0.36688, 0.3704, 0.36443, 0.36411, 0.36221, 0.36185, 0.36498, 0.36202, 0.36553, 0.36574, 0.36507, 0.37335, 0.36256, 0.3648, 0.36324, 0.36253, 0.36685, 0.3644, 0.36463, 0.36584, 0.36426, 0.36134, 0.36175, 0.45788, 0.36568, 0.36196, 0.38364, 0.36164, 0.36331, 0.36346, 0.3683, 0.36544, 0.36245, 0.37051, 0.37092, 0.36741, 0.3695, 0.3651, 0.37195, 0.36315, 0.36425, 0.36904, 0.36828, 0.3648, 0.36763, 0.36895, 0.37272, 0.3749, 0.36753, 0.36573, 0.36845, 0.36886, 0.37096, 0.47625, 0.36339, 0.36255, 0.36368, 0.44639, 0.51442, 0.3673, 0.36637, 0.36885, 0.37285, 0.36987, 0.36631, 0.36485, 0.36259, 0.36217, 0.364, 0.36364, 0.36588, 0.3619, 0.36604, 0.36798, 0.36772, 0.36665, 0.36769, 0.36628, 0.36592, 0.36831, 0.36583, 0.36842, 0.36695, 0.37069, 0.36526, 0.36421, 0.3661, 0.36543, 0.36845, 0.36581, 0.3674, 0.36575, 0.36568, 0.36949, 0.36761, 0.36684, 0.36852, 0.36408, 0.37073, 0.36602, 0.36769, 0.3609, 0.36264, 0.36736, 0.36549, 0.36517, 0.36003, 0.36081, 0.36006, 0.36167, 0.36361, 0.36172, 0.36296, 0.36716, 0.36645, 0.36705, 0.36621, 0.45574, 0.36247, 0.36105, 0.36408, 0.3621, 0.36088, 0.36271, 0.36349, 0.36811, 0.36958, 0.36968, 0.36582, 0.36294, 0.36436, 0.36894, 0.36266, 0.36585, 0.36633, 0.36462, 0.36885, 0.36711, 0.36754, 0.36317, 0.36285, 0.36581, 0.37564, 0.37346, 0.3622, 0.36404, 0.45901, 0.36362, 0.36726, 0.37058, 0.36812, 0.36666, 0.37189, 0.46883, 0.37275, 0.3719, 0.36704, 0.36448, 0.3629, 0.36582, 0.36225, 0.36061, 0.4845, 0.36483, 0.36652, 0.36811, 0.36819, 0.37464, 0.36516, 0.36721, 0.36426, 0.35999, 0.36267, 0.36286, 0.36833, 0.36584, 0.3632, 0.36415, 0.36569, 0.37494, 0.36226, 0.46516, 0.36495, 0.36254, 0.36943, 0.36585, 0.36664, 0.36827, 0.36557, 0.37484, 0.36946, 0.37108, 0.36825, 0.36775, 0.36137, 0.36521, 0.3697, 0.36415, 0.36338, 0.36383, 0.36505, 0.3677, 0.36976, 0.36576, 0.36964, 0.37212, 0.36584, 0.36475, 0.36537, 0.36914, 0.36892, 0.45897, 0.36567, 0.3641, 0.36657, 0.3698, 0.36867, 0.36599, 0.3679, 0.36742, 0.36813, 0.36659, 0.36737, 0.36653, 0.36785, 0.37243, 0.36895, 0.37086, 0.365, 0.36719, 0.37471, 0.36717, 0.3738, 0.37016, 0.37206, 0.3695, 0.36911, 0.36946, 0.36669, 0.36636, 0.3628, 0.3661, 0.36516, 0.36275, 0.3657, 0.3654, 0.36521, 0.3662, 0.4682, 0.36931, 0.3668, 0.37172, 0.37189, 0.36942, 0.37165, 0.37159, 0.37333, 0.37491, 0.37221, 0.36907, 0.37154, 0.37633, 0.36937, 0.36886, 0.36922, 0.36659, 0.36692, 0.36765, 0.36709, 0.3641, 0.36625, 0.36742, 0.36073, 0.36646, 0.36662, 0.36508, 0.37343, 0.36701, 0.3642, 0.36688, 0.36861, 0.36833, 0.36153, 0.36529, 0.36657, 0.36866, 0.37542, 0.36846, 0.36817, 0.36445, 0.36398, 0.36799, 0.36631, 0.3632, 0.36525, 0.36782, 0.36786, 0.37064, 0.36604, 0.36767, 0.36737, 0.36678, 0.36919, 0.36757, 0.36912, 0.36819, 0.46929, 0.37321, 0.37017, 0.4569, 0.36994, 0.37357, 0.36984, 0.57706, 0.37035, 0.37045, 0.36802, 0.36852, 0.36742]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [9.27486, 0.20418, 0.20397, 0.20285, 0.20434, 0.20758, 0.20634, 0.20416, 0.20426, 0.20434, 0.3669, 0.20758, 0.20442, 0.20546, 0.20278, 0.20684, 0.20447, 0.20408, 0.20756, 0.20602, 0.20443, 0.20251, 0.20574, 0.20384, 0.2029, 0.21254, 0.21029, 0.20601, 0.20107, 0.20291, 0.20989, 0.19612, 0.20052, 0.19662, 0.19784, 0.20061, 0.19675, 0.19997, 0.20194, 0.22257, 0.2025, 0.20076, 0.2025, 0.20065, 0.20083, 0.19995, 0.19982, 0.20085, 0.20083, 0.19933, 0.20226, 0.20132, 0.203, 0.19623, 0.1999, 0.19978, 0.1976, 0.19962, 0.19949, 0.19977, 0.19439, 0.19749, 0.19772, 0.19546, 0.19711, 0.19707, 0.19839, 0.19731, 0.20084, 0.19819, 0.2011, 0.1983, 0.19858, 0.1937, 0.29471, 0.19528, 0.19534, 0.19901, 0.20146, 0.19982, 0.19907, 0.20086, 0.20405, 0.19915, 0.2005, 0.19581, 0.19278, 0.19863, 0.19822, 0.1993, 0.1988, 0.19998, 0.2005, 0.19725, 0.20091, 0.19918, 0.19836, 0.2016, 0.19765, 0.19811, 0.19903, 0.19646, 0.19645, 0.19682, 0.28975, 0.19888, 0.19522, 0.21159, 0.19644, 0.19881, 0.19777, 0.20279, 0.19972, 0.19755, 0.20374, 0.20397, 0.20052, 0.20409, 0.20046, 0.20573, 0.19813, 0.19893, 0.20396, 0.20108, 0.1991, 0.20018, 0.20247, 0.20606, 0.20496, 0.20146, 0.20113, 0.20109, 0.20373, 0.20131, 0.30688, 0.19978, 0.19719, 0.19856, 0.27425, 0.34575, 0.20073, 0.20027, 0.20292, 0.20753, 0.20162, 0.19901, 0.19974, 0.19616, 0.19556, 0.19818, 0.19745, 0.20023, 0.19768, 0.1993, 0.20152, 0.20191, 0.20046, 0.19952, 0.19909, 0.20067, 0.20206, 0.20028, 0.2009, 0.20109, 0.20231, 0.20057, 0.19849, 0.2014, 0.19862, 0.20162, 0.1995, 0.20168, 0.19859, 0.20023, 0.20137, 0.19954, 0.19893, 0.20032, 0.19926, 0.20288, 0.20082, 0.20203, 0.1964, 0.19744, 0.20075, 0.19839, 0.19941, 0.19592, 0.19584, 0.19507, 0.19602, 0.19868, 0.19785, 0.19642, 0.20146, 0.20135, 0.20162, 0.20061, 0.28565, 0.19898, 0.19699, 0.20018, 0.1975, 0.19765, 0.19836, 0.20012, 0.20347, 0.20455, 0.20461, 0.20103, 0.1993, 0.20097, 0.20324, 0.19779, 0.20128, 0.20136, 0.19977, 0.20189, 0.20216, 0.19869, 0.19833, 0.19963, 0.20166, 0.21162, 0.2062, 0.19807, 0.19895, 0.29325, 0.19845, 0.1994, 0.20325, 0.20285, 0.20049, 0.20554, 0.30108, 0.20617, 0.20644, 0.20131, 0.20084, 0.19867, 0.20111, 0.19928, 0.19687, 0.31861, 0.20096, 0.20262, 0.20309, 0.20325, 0.20819, 0.20113, 0.20301, 0.19969, 0.19603, 0.19693, 0.19763, 0.2004, 0.20179, 0.19742, 0.19937, 0.20128, 0.20616, 0.19831, 0.29924, 0.19973, 0.19859, 0.20413, 0.20138, 0.20285, 0.20388, 0.20206, 0.20671, 0.20471, 0.20646, 0.20241, 0.20408, 0.19861, 0.20125, 0.20732, 0.20159, 0.20035, 0.20096, 0.20012, 0.20294, 0.20424, 0.20101, 0.20564, 0.2044, 0.2008, 0.19955, 0.20264, 0.2049, 0.20446, 0.293, 0.20181, 0.20025, 0.20162, 0.20369, 0.20417, 0.20115, 0.20265, 0.20363, 0.2044, 0.20297, 0.20322, 0.20046, 0.20222, 0.20483, 0.20332, 0.20676, 0.19998, 0.2015, 0.2054, 0.20246, 0.20845, 0.20406, 0.20619, 0.20592, 0.20453, 0.20274, 0.20274, 0.20162, 0.20007, 0.20274, 0.20276, 0.19873, 0.20293, 0.20198, 0.20198, 0.20314, 0.30676, 0.20607, 0.2049, 0.20889, 0.20967, 0.2072, 0.20824, 0.20768, 0.20857, 0.20862, 0.20898, 0.20615, 0.20827, 0.21418, 0.20637, 0.20388, 0.2067, 0.20272, 0.20336, 0.20429, 0.20148, 0.20112, 0.20264, 0.20322, 0.19861, 0.20195, 0.20314, 0.1996, 0.20578, 0.2036, 0.20073, 0.20362, 0.20652, 0.20449, 0.19954, 0.20273, 0.203, 0.2032, 0.20757, 0.2034, 0.20482, 0.19991, 0.20078, 0.20474, 0.20356, 0.19886, 0.20118, 0.20177, 0.20291, 0.20253, 0.20141, 0.20341, 0.20352, 0.20319, 0.20478, 0.20413, 0.20568, 0.20319, 0.30235, 0.20813, 0.20681, 0.29099, 0.20567, 0.20759, 0.20528, 0.41177, 0.20714, 0.20416, 0.20342, 0.20429, 0.20393]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.48483, 0.17652, 0.17828, 0.17737, 0.17731, 0.18012, 0.18059, 0.17933, 0.18228, 0.17963, 0.17741, 0.17905, 0.17875, 0.18023, 0.17598, 0.17735, 0.17563, 0.1774, 0.17814, 0.17775, 0.1797, 0.17589, 0.17512, 0.17493, 0.17423, 0.17574, 0.17442, 0.17392, 0.17429, 0.18376, 0.17762, 0.17577, 0.17608, 0.17519, 0.17371, 0.17562, 0.1743, 0.17634, 0.17747, 0.1794, 0.17639, 0.1769, 0.17749, 0.17644, 0.17597, 0.17611, 0.17772, 0.17605, 0.17799, 0.1756, 0.17762, 0.17478, 0.17987, 0.17366, 0.17669, 0.17775, 0.17802, 0.17908, 0.17514, 0.17554, 0.17388, 0.17483, 0.17431, 0.17275, 0.17497, 0.17541, 0.17514, 0.17686, 0.17728, 0.17469, 0.17508, 0.17519, 0.17517, 0.17377, 0.17594, 0.17621, 0.17553, 0.17702, 0.18, 0.17602, 0.17593, 0.17864, 0.17997, 0.1755, 0.17822, 0.17772, 0.17671, 0.17725, 0.1778, 0.17809, 0.17954, 0.17593, 0.17541, 0.17441, 0.17679, 0.17798, 0.17778, 0.17724, 0.17552, 0.17811, 0.18023, 0.17981, 0.17557, 0.17566, 0.17625, 0.17625, 0.17558, 0.19425, 0.1762, 0.17767, 0.17763, 0.18372, 0.17971, 0.17752, 0.18218, 0.18258, 0.18042, 0.18083, 0.17934, 0.18263, 0.17612, 0.17585, 0.18209, 0.17892, 0.17504, 0.18056, 0.18269, 0.18216, 0.18105, 0.18046, 0.17895, 0.18001, 0.18287, 0.18048, 0.18107, 0.1792, 0.177, 0.17595, 0.17833, 0.17997, 0.18026, 0.18064, 0.18103, 0.18122, 0.1807, 0.17741, 0.17696, 0.175, 0.17708, 0.17762, 0.17496, 0.17994, 0.17504, 0.17879, 0.18178, 0.1796, 0.18007, 0.18397, 0.18212, 0.18076, 0.18234, 0.18066, 0.18359, 0.18244, 0.18094, 0.18093, 0.17869, 0.18132, 0.18028, 0.18293, 0.17692, 0.181, 0.1778, 0.178, 0.18006, 0.18483, 0.18337, 0.18495, 0.18069, 0.18012, 0.18124, 0.18343, 0.17705, 0.17668, 0.17849, 0.18112, 0.17754, 0.1764, 0.17576, 0.17489, 0.17603, 0.17867, 0.17875, 0.17778, 0.17783, 0.18028, 0.18098, 0.18147, 0.18117, 0.17707, 0.17356, 0.17855, 0.17723, 0.175, 0.17556, 0.17674, 0.17749, 0.17698, 0.17866, 0.17541, 0.17473, 0.17725, 0.17976, 0.17814, 0.17815, 0.17912, 0.17571, 0.18059, 0.18163, 0.17964, 0.17657, 0.1773, 0.17872, 0.18756, 0.18502, 0.17691, 0.17601, 0.1773, 0.17751, 0.17745, 0.18072, 0.17998, 0.17849, 0.18172, 0.17785, 0.18296, 0.17966, 0.18029, 0.17622, 0.17684, 0.17683, 0.17525, 0.17514, 0.17546, 0.17768, 0.17616, 0.17827, 0.17873, 0.18236, 0.17864, 0.17902, 0.17866, 0.17537, 0.17824, 0.17634, 0.17765, 0.17745, 0.17691, 0.17855, 0.17773, 0.1776, 0.17553, 0.17612, 0.17682, 0.17445, 0.17573, 0.17792, 0.17697, 0.17758, 0.17799, 0.18179, 0.17862, 0.17828, 0.17902, 0.17716, 0.17378, 0.17466, 0.17969, 0.17531, 0.17449, 0.1762, 0.17533, 0.17786, 0.17799, 0.1739, 0.17695, 0.17997, 0.17727, 0.17594, 0.17599, 0.17877, 0.17835, 0.17768, 0.17619, 0.1761, 0.17947, 0.18082, 0.17999, 0.17973, 0.18161, 0.17878, 0.18107, 0.17669, 0.17787, 0.17714, 0.17987, 0.17952, 0.18139, 0.1814, 0.17879, 0.17819, 0.17967, 0.17842, 0.18204, 0.17981, 0.18039, 0.1779, 0.17786, 0.18096, 0.17907, 0.17853, 0.17539, 0.17682, 0.17666, 0.17653, 0.17793, 0.17688, 0.1782, 0.17909, 0.17471, 0.17743, 0.17531, 0.17878, 0.17697, 0.1762, 0.17958, 0.17827, 0.17938, 0.17923, 0.17797, 0.1763, 0.17776, 0.18097, 0.17754, 0.18018, 0.17934, 0.1806, 0.1751, 0.17845, 0.18106, 0.17667, 0.17809, 0.17911, 0.17624, 0.17874, 0.1795, 0.17661, 0.18214, 0.18117, 0.17941, 0.17482, 0.17595, 0.17616, 0.17509, 0.17725, 0.17932, 0.18085, 0.18292, 0.17986, 0.17974, 0.17799, 0.17756, 0.17851, 0.17744, 0.17724, 0.17992, 0.18197, 0.18128, 0.1816, 0.17718, 0.1781, 0.18028, 0.17962, 0.18211, 0.17904, 0.18027, 0.179, 0.1805, 0.18514, 0.18111, 0.17608, 0.18024, 0.1833, 0.1823, 0.1797, 0.17902, 0.18251, 0.18061, 0.17877, 0.17926]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.60562, 0.0038, 0.00384, 0.00379, 0.00392, 0.00392, 0.00391, 0.00387, 0.00391, 0.00397, 0.00392, 0.00405, 0.00383, 0.00388, 0.00387, 0.0042, 0.00394, 0.00394, 0.00387, 0.00379, 0.00413, 0.00393, 0.00403, 0.00383, 0.00384, 0.004, 0.0044, 0.00355, 0.00419, 0.00392, 0.00399, 0.00394, 0.0037, 0.00364, 0.00369, 0.00383, 0.00379, 0.00369, 0.0038, 0.00364, 0.00377, 0.00393, 0.00365, 0.00367, 0.00383, 0.00366, 0.00382, 0.00371, 0.00355, 0.00439, 0.00359, 0.00368, 0.00365, 0.00383, 0.00363, 0.00374, 0.00373, 0.00378, 0.00373, 0.00352, 0.00362, 0.0036, 0.00343, 0.00349, 0.00382, 0.00374, 0.00356, 0.00374, 0.00365, 0.00391, 0.0037, 0.00375, 0.00369, 0.00366, 0.00397, 0.00372, 0.00358, 0.00365, 0.00406, 0.00355, 0.00339, 0.00398, 0.00424, 0.0036, 0.00363, 0.00389, 0.00371, 0.00377, 0.00362, 0.00383, 0.00373, 0.0037, 0.00388, 0.00356, 0.00358, 0.00363, 0.00387, 0.00375, 0.00383, 0.00372, 0.00369, 0.00374, 0.00411, 0.00364, 0.0039, 0.00376, 0.00383, 0.00364, 0.00379, 0.00378, 0.00364, 0.00365, 0.00392, 0.00347, 0.00361, 0.00377, 0.00359, 0.00364, 0.00383, 0.00375, 0.00368, 0.00367, 0.0041, 0.00379, 0.00359, 0.00366, 0.00379, 0.00376, 0.00387, 0.00368, 0.00361, 0.00375, 0.00401, 0.0038, 0.00393, 0.00377, 0.00358, 0.00402, 0.00479, 0.00399, 0.00374, 0.00392, 0.00379, 0.00391, 0.00355, 0.00378, 0.00356, 0.00362, 0.0036, 0.00351, 0.00348, 0.00422, 0.00355, 0.00359, 0.00351, 0.00373, 0.00362, 0.00377, 0.00378, 0.00386, 0.0037, 0.00367, 0.00361, 0.0038, 0.00392, 0.00338, 0.00354, 0.00357, 0.00375, 0.00369, 0.0038, 0.0036, 0.00386, 0.00388, 0.00354, 0.00367, 0.00381, 0.00354, 0.00366, 0.0038, 0.00367, 0.00378, 0.00363, 0.00368, 0.00358, 0.00359, 0.00373, 0.00355, 0.00402, 0.00361, 0.00364, 0.00369, 0.0035, 0.00356, 0.00387, 0.00375, 0.00381, 0.0038, 0.00396, 0.00375, 0.03419, 0.00346, 0.00373, 0.00413, 0.0035, 0.00359, 0.00362, 0.00344, 0.00367, 0.00349, 0.00362, 0.00369, 0.00353, 0.00388, 0.00372, 0.00358, 0.0036, 0.00347, 0.00344, 0.00368, 0.00381, 0.00355, 0.00366, 0.0035, 0.00362, 0.00372, 0.0037, 0.00382, 0.00365, 0.00381, 0.00385, 0.00362, 0.00358, 0.00369, 0.00374, 0.00368, 0.00355, 0.00377, 0.00348, 0.00351, 0.00355, 0.00339, 0.00354, 0.00335, 0.00357, 0.00367, 0.00363, 0.00377, 0.00357, 0.00363, 0.00374, 0.00361, 0.00358, 0.00354, 0.00336, 0.00361, 0.00371, 0.00365, 0.00354, 0.00394, 0.00379, 0.00378, 0.00379, 0.00401, 0.00398, 0.00384, 0.00395, 0.0042, 0.00424, 0.00421, 0.00426, 0.00442, 0.00415, 0.00404, 0.0043, 0.00406, 0.00434, 0.00442, 0.00416, 0.0043, 0.00409, 0.00403, 0.00412, 0.004, 0.00407, 0.00448, 0.00415, 0.00407, 0.0041, 0.0041, 0.00402, 0.00417, 0.00421, 0.00402, 0.00399, 0.00398, 0.00422, 0.00414, 0.00414, 0.00417, 0.00412, 0.004, 0.00405, 0.00393, 0.00399, 0.00391, 0.00392, 0.00387, 0.00417, 0.00413, 0.00408, 0.004, 0.00415, 0.00409, 0.00421, 0.00397, 0.00405, 0.00396, 0.00405, 0.00404, 0.00407, 0.00408, 0.00399, 0.004, 0.00392, 0.00412, 0.00432, 0.00438, 0.00426, 0.00415, 0.00429, 0.00422, 0.00401, 0.00419, 0.0041, 0.00398, 0.00406, 0.00453, 0.00398, 0.00413, 0.00404, 0.00406, 0.00404, 0.00404, 0.0041, 0.00409, 0.00402, 0.00399, 0.0041, 0.00413, 0.00436, 0.00417, 0.00418, 0.00424, 0.00423, 0.00429, 0.00425, 0.00417, 0.00427, 0.00432, 0.00421, 0.00425, 0.00421, 0.00433, 0.00423, 0.00439, 0.00428, 0.00423, 0.00424, 0.0041, 0.00423, 0.00424, 0.00433, 0.00424, 0.00436, 0.0043, 0.00407, 0.00429, 0.0041, 0.00429, 0.00431, 0.00428, 0.0043, 0.00425, 0.00416, 0.00427, 0.00405, 0.00443, 0.00417, 0.0042, 0.00449, 0.00406, 0.004, 0.00406, 0.0042, 0.00421, 0.00409, 0.00421, 0.00421, 0.00413]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 5e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [5.81083, 0.0018, 0.00179, 0.00169, 0.00153, 0.00181, 0.00157, 0.00183, 0.00159, 0.00178, 0.00159, 0.00178, 0.00153, 0.00181, 0.0016, 0.0018, 0.00158, 0.00176, 0.00155, 0.00182, 0.00162, 0.00179, 0.00159, 0.00178, 0.0016, 0.00183, 0.00159, 0.00181, 0.0016, 0.00181, 0.00161, 0.0018, 0.00156, 0.00165, 0.0016, 0.00177, 0.00157, 0.00177, 0.00159, 0.00175, 0.00158, 0.00178, 0.00159, 0.00182, 0.00158, 0.00177, 0.00158, 0.00177, 0.00159, 0.00179, 0.00155, 0.00183, 0.00158, 0.00178, 0.00156, 0.00181, 0.00154, 0.0018, 0.00154, 0.00178, 0.00159, 0.00181, 0.00157, 0.00181, 0.00155, 0.00183, 0.00159, 0.0018, 0.00155, 0.00179, 0.00158, 0.00181, 0.00159, 0.00179, 0.00153, 0.00178, 0.00157, 0.00178, 0.00156, 0.00176, 0.00156, 0.00179, 0.00157, 0.00182, 0.00152, 0.00181, 0.00152, 0.00183, 0.00157, 0.00179, 0.00159, 0.00187, 0.00159, 0.00182, 0.00156, 0.0018, 0.00161, 0.0018, 0.00157, 0.00176, 0.00159, 0.00179, 0.00157, 0.00182, 0.00158, 0.0018, 0.0016, 0.00182, 0.00159, 0.00172, 0.00157, 0.00179, 0.00154, 0.00166, 0.00158, 0.00176, 0.00159, 0.00184, 0.00156, 0.00179, 0.00157, 0.00174, 0.00157, 0.00173, 0.00157, 0.0018, 0.00159, 0.00181, 0.00156, 0.00183, 0.00157, 0.00181, 0.00158, 0.00179, 0.00157, 0.00184, 0.00158, 0.00174, 0.00163, 0.00175, 0.00158, 0.0018, 0.00152, 0.00183, 0.00158, 0.00174, 0.00159, 0.00179, 0.00155, 0.00182, 0.00157, 0.0018, 0.00159, 0.00183, 0.00156, 0.00181, 0.00158, 0.00176, 0.00158, 0.00176, 0.00156, 0.00178, 0.00158, 0.00181, 0.00153, 0.0018, 0.00155, 0.0018, 0.0016, 0.0019, 0.0016, 0.00175, 0.0016, 0.0018, 0.00153, 0.00178, 0.00158, 0.0018, 0.00156, 0.00172, 0.00159, 0.00182, 0.00157, 0.00175, 0.00157, 0.00173, 0.00156, 0.00186, 0.00158, 0.00178, 0.00158, 0.00188, 0.00159, 0.00181, 0.00153, 0.00175, 0.00155, 0.00181, 0.00156, 0.00181, 0.00177, 0.00157, 0.00162, 0.00165, 0.00173, 0.00157, 0.00173, 0.00165, 0.00167, 0.00151, 0.00172, 0.00167, 0.00174, 0.00157, 0.00168, 0.00168, 0.00174, 0.00157, 0.00175, 0.00166, 0.00174, 0.00154, 0.00174, 0.00167, 0.00171, 0.00159, 0.00174, 0.00165, 0.00173, 0.00159, 0.00174, 0.00162, 0.00175, 0.00157, 0.00174, 0.00167, 0.00172, 0.00156, 0.00174, 0.00164, 0.00175, 0.00154, 0.00161, 0.0016, 0.00174, 0.00156, 0.00179, 0.00167, 0.00167, 0.00155, 0.00175, 0.00167, 0.00173, 0.00158, 0.00176, 0.00166, 0.00173, 0.00157, 0.00173, 0.00161, 0.00176, 0.0016, 0.00168, 0.00162, 0.00174, 0.00158, 0.00174, 0.00167, 0.00174, 0.00158, 0.00168, 0.00161, 0.00175, 0.00159, 0.00173, 0.00168, 0.00175, 0.00158, 0.00174, 0.00163, 0.00176, 0.00153, 0.00175, 0.00168, 0.00168, 0.00153, 0.00172, 0.00165, 0.00175, 0.00159, 0.00174, 0.00164, 0.00176, 0.00153, 0.00171, 0.00162, 0.00173, 0.00156, 0.00174, 0.00165, 0.00168, 0.00158, 0.00174, 0.00167, 0.00176, 0.00158, 0.00175, 0.00167, 0.00174, 0.00158, 0.00168, 0.00166, 0.00173, 0.00157, 0.00176, 0.00161, 0.00173, 0.00159, 0.00178, 0.00165, 0.00174, 0.00156, 0.00167, 0.00163, 0.00165, 0.00158, 0.00173, 0.00162, 0.00176, 0.00157, 0.00173, 0.00166, 0.00173, 0.0016, 0.0018, 0.00165, 0.00172, 0.00159, 0.00168, 0.00165, 0.00175, 0.00154, 0.00171, 0.00164, 0.00169, 0.00153, 0.00175, 0.00166, 0.00175, 0.00159, 0.00176, 0.00164, 0.00172, 0.00159, 0.00169, 0.00166, 0.00173, 0.00153, 0.00167, 0.00164, 0.00172, 0.00159, 0.00167, 0.00168, 0.00175, 0.00157, 0.00173, 0.00167, 0.00172, 0.0016, 0.00173, 0.00166, 0.00175, 0.00153, 0.00174, 0.00163, 0.00172, 0.00157, 0.00167, 0.00165, 0.00171, 0.00159, 0.00175, 0.00166, 0.00166, 0.00158, 0.00166, 0.00164, 0.00167, 0.00157, 0.0017, 0.00168, 0.00169, 0.00158, 0.00176, 0.00168, 0.00172, 0.00157, 0.00173, 0.00167]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00181, 0.00152, 0.00153, 0.0015, 0.00157, 0.00156, 0.00152, 0.00157, 0.00162, 0.0015, 0.00152, 0.00155, 0.00152, 0.00155, 0.00155, 0.00161, 0.00151, 0.00151, 0.00196, 0.0015, 0.00161, 0.0015, 0.00162, 0.00161, 0.00157, 0.00151, 0.0015, 0.0015, 0.00156, 0.00153, 0.00171, 0.00252, 0.00165, 0.0018, 0.00159, 0.00153, 0.00157, 0.00159, 0.00159, 0.00157, 0.00156, 0.00163, 0.00152, 0.0015, 0.00163, 0.00153, 0.00149, 0.00156, 0.00156, 0.00152, 0.00157, 0.00152, 0.0016, 0.00159, 0.00155, 0.00157, 0.00157, 0.00156, 0.00151, 0.00156, 0.00152, 0.00151, 0.00157, 0.00157, 0.00163, 0.00153, 0.00158, 0.00155, 0.00149, 0.00161, 0.0015, 0.00156, 0.00151, 0.00162, 0.00158, 0.00148, 0.00156, 0.0015, 0.00157, 0.00151, 0.00155, 0.00155, 0.00161, 0.0027, 0.00157, 0.00156, 0.00156, 0.00151, 0.00156, 0.00149, 0.00158, 0.0015, 0.00152, 0.00156, 0.00155, 0.0024, 0.00156, 0.0016, 0.00156, 0.0015, 0.0016, 0.00155, 0.00151, 0.00154, 0.00158, 0.0015, 0.0015, 0.00155, 0.00156, 0.00155, 0.00157, 0.0015, 0.0015, 0.00155, 0.00157, 0.00155, 0.00157, 0.0015, 0.00157, 0.00155, 0.00155, 0.0015, 0.00164, 0.0016, 0.00151, 0.0015, 0.00165, 0.00151, 0.00157, 0.00157, 0.00158, 0.00154, 0.00157, 0.0016, 0.0016, 0.00149, 0.00154, 0.00156, 0.00333, 0.00159, 0.00153, 0.00149, 0.00149, 0.00166, 0.00165, 0.00158, 0.00149, 0.00155, 0.00152, 0.00155, 0.00156, 0.00152, 0.00155, 0.00156, 0.00164, 0.00155, 0.00156, 0.00152, 0.00166, 0.00153, 0.0015, 0.0015, 0.00155, 0.00156, 0.00158, 0.00149, 0.00165, 0.00155, 0.0015, 0.0015, 0.0015, 0.00154, 0.00155, 0.00165, 0.00156, 0.00155, 0.0015, 0.00148, 0.00154, 0.00156, 0.00156, 0.0015, 0.00148, 0.00157, 0.00152, 0.0015, 0.00149, 0.00157, 0.00149, 0.00149, 0.0015, 0.0028, 0.0015, 0.00151, 0.00157, 0.00155, 0.00148, 0.0015, 0.00169, 0.00149, 0.0015, 0.00159, 0.00155, 0.00149, 0.0015, 0.00148, 0.00149, 0.00154, 0.00155, 0.00149, 0.00147, 0.00149, 0.00156, 0.00148, 0.00146, 0.00151, 0.00152, 0.00147, 0.00147, 0.00147, 0.00155, 0.00147, 0.00148, 0.00144, 0.0015, 0.0015, 0.00159, 0.00156, 0.00149, 0.00151, 0.0016, 0.00149, 0.0015, 0.00154, 0.0015, 0.00147, 0.00147, 0.00154, 0.00156, 0.00153, 0.0015, 0.0015, 0.002, 0.00151, 0.00246, 0.0015, 0.00147, 0.00144, 0.00148, 0.00171, 0.00148, 0.0015, 0.00157, 0.00174, 0.00156, 0.00157, 0.00148, 0.00147, 0.00149, 0.00148, 0.0015, 0.00148, 0.00151, 0.00158, 0.00149, 0.00147, 0.00153, 0.00151, 0.00154, 0.00148, 0.00157, 0.00157, 0.00148, 0.0016, 0.00153, 0.00155, 0.00156, 0.00157, 0.00149, 0.00154, 0.00148, 0.00151, 0.00149, 0.00155, 0.00148, 0.00155, 0.00155, 0.0015, 0.00149, 0.0015, 0.00149, 0.00153, 0.00164, 0.0016, 0.0015, 0.00153, 0.00149, 0.00158, 0.00154, 0.00149, 0.00154, 0.00165, 0.00151, 0.00148, 0.00158, 0.00157, 0.00158, 0.0015, 0.00149, 0.00154, 0.00152, 0.00155, 0.00158, 0.00149, 0.00157, 0.0015, 0.00158, 0.00163, 0.00159, 0.00158, 0.00159, 0.00157, 0.00157, 0.0015, 0.00151, 0.00151, 0.00154, 0.00154, 0.00159, 0.00155, 0.00155, 0.00148, 0.00198, 0.00154, 0.00149, 0.00156, 0.00151, 0.00157, 0.00149, 0.00148, 0.00151, 0.00154, 0.00153, 0.00148, 0.00151, 0.00149, 0.0015, 0.00155, 0.00155, 0.00151, 0.00156, 0.00154, 0.0015, 0.0015, 0.00151, 0.00157, 0.00156, 0.00158, 0.0015, 0.00155, 0.00148, 0.00153, 0.00151, 0.0015, 0.0015, 0.00152, 0.00151, 0.00156, 0.00158, 0.00151, 0.0015, 0.00149, 0.00156, 0.00156, 0.00157, 0.0015, 0.00148, 0.00158, 0.00158, 0.00156, 0.00155, 0.00154, 0.00165, 0.00162, 0.00157, 0.00166, 0.0015, 0.00156, 0.00155, 0.00152, 0.00152, 0.00154, 0.0015, 0.00153, 0.0016, 0.0015, 0.00151, 0.00152, 0.00155, 0.00155]}, "optimizer-unscale-and-check-inf-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.60633, 0.00085, 0.00071, 0.0006, 0.00062, 0.0006, 0.00062, 0.00062, 0.00063, 0.00059, 0.00063, 0.00062, 0.00063, 0.00063, 0.00063, 0.00068, 0.00062, 0.00063, 0.00065, 0.00064, 0.00064, 0.0006, 0.00063, 0.00064, 0.00063, 0.00061, 0.00062, 0.00062, 0.00063, 0.00061, 0.0007, 0.00092, 0.00063, 0.00071, 0.00063, 0.00069, 0.00063, 0.00062, 0.00063, 0.00063, 0.00064, 0.0006, 0.00061, 0.00064, 0.00062, 0.00063, 0.00061, 0.00065, 0.00062, 0.00062, 0.0006, 0.00062, 0.00067, 0.00061, 0.00062, 0.00062, 0.00061, 0.00063, 0.00061, 0.00061, 0.0006, 0.00062, 0.00061, 0.00062, 0.00062, 0.00062, 0.00064, 0.00061, 0.00062, 0.00063, 0.00061, 0.00062, 0.00061, 0.00065, 0.00063, 0.0006, 0.0006, 0.0006, 0.00064, 0.00063, 0.00064, 0.0006, 0.00061, 0.00077, 0.00062, 0.00062, 0.00062, 0.00061, 0.00061, 0.00064, 0.00062, 0.0006, 0.00062, 0.00062, 0.00059, 0.00067, 0.00061, 0.00065, 0.0006, 0.00061, 0.00063, 0.00062, 0.00063, 0.00063, 0.00062, 0.0006, 0.00061, 0.00062, 0.00062, 0.0006, 0.00063, 0.00061, 0.0006, 0.0006, 0.00059, 0.00061, 0.0006, 0.00063, 0.00062, 0.00062, 0.00062, 0.00059, 0.00063, 0.0006, 0.00062, 0.00062, 0.00062, 0.00059, 0.00062, 0.00063, 0.0006, 0.00061, 0.0006, 0.00067, 0.00069, 0.00061, 0.00061, 0.00063, 0.00074, 0.0006, 0.00061, 0.00061, 0.00061, 0.00066, 0.00071, 0.00062, 0.00061, 0.0006, 0.00061, 0.00063, 0.0006, 0.00063, 0.00062, 0.00063, 0.00061, 0.00063, 0.00063, 0.00063, 0.00064, 0.00063, 0.00065, 0.00064, 0.00062, 0.00061, 0.00063, 0.00061, 0.00062, 0.00061, 0.00062, 0.00062, 0.00061, 0.00063, 0.00063, 0.00064, 0.00063, 0.00063, 0.00062, 0.00063, 0.00061, 0.00064, 0.00067, 0.0006, 0.00061, 0.00062, 0.00071, 0.00062, 0.00059, 0.00063, 0.00062, 0.0006, 0.00061, 0.00065, 0.00061, 0.00062, 0.00063, 0.00063, 0.00062, 0.00061, 0.00065, 0.00061, 0.00059, 0.0006, 0.00062, 0.0006, 0.00063, 0.00063, 0.0006, 0.00061, 0.00059, 0.00062, 0.00062, 0.0006, 0.00064, 0.00058, 0.00059, 0.00063, 0.00059, 0.0006, 0.00059, 0.00061, 0.00063, 0.00063, 0.0006, 0.0006, 0.00062, 0.0006, 0.00061, 0.00062, 0.00059, 0.00063, 0.0006, 0.00063, 0.0006, 0.00063, 0.00061, 0.00076, 0.0006, 0.00059, 0.00059, 0.00059, 0.0006, 0.00063, 0.00067, 0.00062, 0.00096, 0.00064, 0.00063, 0.00065, 0.00059, 0.00066, 0.00059, 0.0006, 0.00063, 0.00062, 0.00061, 0.00063, 0.00062, 0.00063, 0.00063, 0.00063, 0.0006, 0.00064, 0.00062, 0.00067, 0.00059, 0.00061, 0.00062, 0.00061, 0.00062, 0.0006, 0.0006, 0.00063, 0.00062, 0.00066, 0.00063, 0.00062, 0.00061, 0.00062, 0.00063, 0.00065, 0.00063, 0.00062, 0.00064, 0.00064, 0.00062, 0.00061, 0.00062, 0.00065, 0.00062, 0.00062, 0.00059, 0.00063, 0.00064, 0.0006, 0.00063, 0.00063, 0.00062, 0.00064, 0.00061, 0.00063, 0.00061, 0.0006, 0.00063, 0.00064, 0.00067, 0.00066, 0.00063, 0.00062, 0.00061, 0.00063, 0.00061, 0.00063, 0.00062, 0.00062, 0.00063, 0.00064, 0.00063, 0.00061, 0.00063, 0.00062, 0.00066, 0.00062, 0.00062, 0.00062, 0.00062, 0.00063, 0.00066, 0.00062, 0.00067, 0.00068, 0.00094, 0.00061, 0.00091, 0.00064, 0.00062, 0.00061, 0.00062, 0.00062, 0.00061, 0.00062, 0.00061, 0.00063, 0.00059, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00059, 0.00066, 0.00062, 0.00062, 0.0006, 0.00062, 0.00061, 0.00063, 0.00062, 0.00062, 0.00062, 0.00059, 0.0006, 0.00061, 0.0006, 0.00062, 0.00063, 0.00063, 0.00061, 0.00063, 0.00064, 0.00061, 0.00062, 0.00062, 0.00062, 0.00093, 0.00063, 0.00063, 0.00063, 0.00062, 0.00059, 0.00061, 0.00062, 0.00062, 0.00064, 0.00062, 0.00064, 0.00063, 0.00064, 0.00064, 0.00063, 0.00062, 0.00063, 0.00062, 0.00062, 0.00066, 0.00064, 0.00074, 0.00063, 0.00063, 0.00062]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.60837, 0.00254, 0.00241, 0.00228, 0.01048, 0.01037, 0.01037, 0.01043, 0.01058, 0.01048, 0.01043, 0.01043, 0.01041, 0.0104, 0.01041, 0.01065, 0.01035, 0.01034, 0.01163, 0.01037, 0.01065, 0.01028, 0.01071, 0.01072, 0.01046, 0.0103, 0.01034, 0.01036, 0.01049, 0.01035, 0.01149, 0.01326, 0.01057, 0.0123, 0.01043, 0.0108, 0.01045, 0.01043, 0.01054, 0.01044, 0.01042, 0.01047, 0.01038, 0.01036, 0.01051, 0.01045, 0.01031, 0.01066, 0.01039, 0.01038, 0.01045, 0.01039, 0.01082, 0.01041, 0.01037, 0.01039, 0.0104, 0.01052, 0.01036, 0.01042, 0.01043, 0.01041, 0.01041, 0.01038, 0.01048, 0.01055, 0.01067, 0.01037, 0.01034, 0.01046, 0.01031, 0.01091, 0.01032, 0.01102, 0.0105, 0.01027, 0.01037, 0.01029, 0.01047, 0.0104, 0.01046, 0.01038, 0.01047, 0.01178, 0.0104, 0.01074, 0.01048, 0.01035, 0.01038, 0.01049, 0.01045, 0.01029, 0.0104, 0.01038, 0.01035, 0.01254, 0.01037, 0.01078, 0.01036, 0.01033, 0.01045, 0.01036, 0.01034, 0.01037, 0.01041, 0.01036, 0.01033, 0.01079, 0.01038, 0.01041, 0.01023, 0.01009, 0.01031, 0.01035, 0.01038, 0.01037, 0.01044, 0.01035, 0.01041, 0.01038, 0.01021, 0.0103, 0.01049, 0.01051, 0.01036, 0.01032, 0.01054, 0.01033, 0.01041, 0.01043, 0.01041, 0.01037, 0.01014, 0.01109, 0.01092, 0.01032, 0.01033, 0.01042, 0.02222, 0.01043, 0.01036, 0.01031, 0.01034, 0.01109, 0.01102, 0.01041, 0.01027, 0.01035, 0.0103, 0.01041, 0.01036, 0.01039, 0.01035, 0.01041, 0.01048, 0.01069, 0.01042, 0.01035, 0.01064, 0.01041, 0.01045, 0.01034, 0.01039, 0.01039, 0.01043, 0.01033, 0.01133, 0.01034, 0.01033, 0.01034, 0.01031, 0.01035, 0.0104, 0.01052, 0.01043, 0.01047, 0.01036, 0.01029, 0.01035, 0.01042, 0.01057, 0.0103, 0.0103, 0.01039, 0.0109, 0.0103, 0.0103, 0.0105, 0.01036, 0.01034, 0.01033, 0.01214, 0.01032, 0.0103, 0.01039, 0.01085, 0.01031, 0.01031, 0.01064, 0.01141, 0.01028, 0.01048, 0.01035, 0.01021, 0.01033, 0.01032, 0.01023, 0.01127, 0.01075, 0.01024, 0.01023, 0.01023, 0.01033, 0.01036, 0.01017, 0.01034, 0.01026, 0.01036, 0.01019, 0.01026, 0.01033, 0.01163, 0.0102, 0.01023, 0.01031, 0.01033, 0.01042, 0.01049, 0.01036, 0.01032, 0.01053, 0.01033, 0.01034, 0.01037, 0.01037, 0.01078, 0.01026, 0.01052, 0.01028, 0.01028, 0.01025, 0.01028, 0.01147, 0.01035, 0.01173, 0.01035, 0.01038, 0.01027, 0.01027, 0.01065, 0.01023, 0.01027, 0.01043, 0.01054, 0.01038, 0.01054, 0.01028, 0.01026, 0.0103, 0.01038, 0.0104, 0.0103, 0.0104, 0.01114, 0.01027, 0.01028, 0.01042, 0.01027, 0.01037, 0.01028, 0.01061, 0.01066, 0.01034, 0.0108, 0.01035, 0.01037, 0.01038, 0.01034, 0.01138, 0.01141, 0.01027, 0.01041, 0.01039, 0.01039, 0.01031, 0.01042, 0.01036, 0.01077, 0.01045, 0.01035, 0.0105, 0.01039, 0.01057, 0.01041, 0.01033, 0.01039, 0.01029, 0.0106, 0.01032, 0.01029, 0.01034, 0.01044, 0.01035, 0.01034, 0.0111, 0.01066, 0.01041, 0.0103, 0.01025, 0.01038, 0.01037, 0.01064, 0.0105, 0.0103, 0.01048, 0.01051, 0.01052, 0.01041, 0.0104, 0.01041, 0.01044, 0.01036, 0.01043, 0.01038, 0.01034, 0.01033, 0.01126, 0.01037, 0.01044, 0.01078, 0.01116, 0.01162, 0.01139, 0.01058, 0.0105, 0.01061, 0.01053, 0.01057, 0.01058, 0.01058, 0.01057, 0.0106, 0.01051, 0.01054, 0.01067, 0.0109, 0.01057, 0.01057, 0.01057, 0.01051, 0.01063, 0.01186, 0.0105, 0.01054, 0.01053, 0.01061, 0.01062, 0.01089, 0.01057, 0.0106, 0.01047, 0.01071, 0.0105, 0.01049, 0.01052, 0.01054, 0.01057, 0.0106, 0.01078, 0.01062, 0.01067, 0.01052, 0.01059, 0.01061, 0.01212, 0.01052, 0.01054, 0.01063, 0.0106, 0.01057, 0.01098, 0.01059, 0.01077, 0.01074, 0.01076, 0.01115, 0.01053, 0.01121, 0.01063, 0.01056, 0.01057, 0.01061, 0.01059, 0.01061, 0.01076, 0.01059, 0.01075, 0.01057, 0.01058, 0.01057]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.89393, 10.90229, 10.90382, 10.89922, 10.90215, 10.87439, 10.80338, 10.63346, 10.44036, 10.2933, 10.02711, 10.16747, 10.13781, 9.86192, 9.97684, 9.67806, 9.59835, 9.78149, 9.50324, 9.44529, 9.35262, 9.25422, 9.27971, 9.09386, 9.28651, 9.15722, 9.24673, 9.26197, 9.39815, 9.08902, 9.03506, 9.14524, 9.15344, 8.76086, 8.82546, 8.85801, 8.78594, 8.83766, 8.7627, 8.8693, 8.76505, 8.95513, 8.94138, 8.60415, 8.49526, 8.5414, 8.6052, 8.49378, 8.54563, 8.69589, 8.47931, 8.31047, 8.34191, 8.33761, 8.38482, 8.03117, 8.21698, 8.01005, 8.36597, 8.35171, 8.1238, 8.08903, 8.03892, 7.85884, 7.86204, 7.76178, 7.63785, 8.03256, 7.82491, 7.57767, 7.87018, 7.89663, 7.66576, 7.41891, 7.57945, 7.45949, 7.58407, 7.3365, 7.75478, 7.39312, 7.46005, 7.32601, 7.32261, 7.53324, 7.28432, 7.3906, 7.10455, 7.1031, 7.135, 7.2333, 6.91495, 7.07308, 7.17321, 7.08148, 6.95568, 6.83552, 7.07146, 7.13597, 6.77633, 6.6537, 6.79923, 6.81094, 6.80156, 6.80623, 6.72479, 6.46997, 6.7029, 6.67891, 6.50414, 6.69017, 6.80201, 6.66742, 6.78223, 6.74908, 6.68039, 6.55851, 6.65127, 6.45882, 6.71595, 6.3003, 6.29947, 6.35127, 6.43626, 6.39728, 6.5005, 6.33652, 6.38489, 6.2805, 6.24364, 6.44007, 6.36837, 6.36408, 6.20465, 6.19665, 6.27951, 6.42484, 6.24039, 6.18602, 6.21368, 6.14857, 6.09651, 6.10359, 6.28963, 6.44182, 6.28988, 6.33247, 6.13546, 6.21108, 6.0349, 6.06273, 5.987, 6.28025, 6.22641, 5.99808, 5.81837, 6.16027, 5.88364, 6.139, 5.82189, 6.19536, 6.17777, 6.11785, 5.96408, 6.14649, 5.9753, 6.22609, 5.92665, 5.82529, 5.80636, 5.7182, 6.04353, 6.02584, 6.092, 5.9119, 6.06757, 5.99273, 6.02669, 6.01523, 5.97662, 5.86429, 5.97653, 5.6431, 5.7275, 5.9135, 5.8664, 5.88797, 5.78842, 5.86055, 5.75215, 5.58542, 5.74699, 5.6532, 5.85871, 5.63063, 5.7325, 5.73883, 5.92312, 5.66992, 5.87123, 5.76346, 5.89613, 5.35339, 5.91985, 5.89554, 5.87623, 5.43362, 5.42829, 5.64744, 5.61678, 5.5103, 5.59917, 5.6988, 5.49854, 5.77013, 5.53314, 5.61954, 5.64553, 5.64008, 5.53513, 5.63528, 5.69717, 5.71522, 5.60874, 5.6802, 5.39435, 5.70021, 5.64782, 5.44435, 5.60824, 5.65007, 5.57098, 5.36362, 5.55798, 5.50433, 5.50082, 5.39457, 5.57452, 5.62082, 5.40855, 5.54177, 5.50319, 5.34993, 5.52256, 5.42475, 5.457, 5.33418, 5.08125, 5.49351, 5.58285, 5.72877, 5.42977, 5.613, 5.64847, 5.2484, 5.28756, 5.41008, 5.40961, 5.34061, 5.51276, 5.19903, 5.31256, 5.26266, 5.3907, 5.27539, 5.46188, 5.55243, 5.32608, 5.4523, 5.34935, 5.085, 5.3281, 5.26395, 5.31744, 5.12555, 5.28677, 5.2827, 5.486, 5.17172, 5.28031, 5.22155, 5.37027, 4.99359, 4.92973, 5.33403, 5.3997, 5.23719, 5.33061, 5.11473, 5.1717, 5.27268, 5.07733, 5.2767, 5.0858, 5.35129, 5.2583, 5.16657, 5.25468, 5.05243, 5.32453, 5.06278, 5.03705, 5.15134, 5.12068, 5.28265, 5.15883, 5.28883, 5.10618, 5.10727, 5.2621, 5.33107, 5.26622, 5.20237, 5.15543, 5.29779, 4.95636, 5.21799, 5.10164, 5.30924, 5.18679, 5.19599, 5.12317, 4.99367, 5.00306, 5.23171, 5.32198, 5.10695, 5.0647, 4.92646, 5.13309, 5.12718, 4.93681, 5.34691, 5.03142, 5.11047, 5.16889, 5.01087, 5.07032, 5.07588, 5.00122, 5.08773, 5.16951, 4.98692, 5.18998, 4.93899, 4.92741, 5.07395, 5.00085, 4.91692, 4.78186, 4.94917, 5.12365, 5.02541, 5.02437, 5.33759, 4.96582, 5.00145, 5.05138, 4.81301, 4.74456, 5.00203, 5.04679, 4.88367, 4.95882, 5.05212, 5.03024, 4.82289, 4.89705, 4.91162, 4.83722, 4.75468, 5.01694, 4.75625, 5.21634, 4.78922, 4.99899, 4.74083, 4.79117, 4.82499, 4.65555, 4.66118, 4.84502, 4.812, 4.80818, 4.93087, 4.88819, 4.92996, 4.77146, 4.88927, 4.73848, 4.91779, 4.96467, 4.87947, 4.7104, 4.78793, 4.90438, 4.71479, 4.86815, 4.69617, 4.69095, 4.65249]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.89393, 10.90229, 10.90382, 10.89922, 10.90215, 10.87439, 10.80338, 10.63346, 10.44036, 10.2933, 10.02711, 10.16747, 10.13781, 9.86192, 9.97684, 9.67806, 9.59835, 9.78149, 9.50324, 9.44529, 9.35262, 9.25422, 9.27971, 9.09386, 9.28651, 9.15722, 9.24673, 9.26197, 9.39815, 9.08902, 9.03506, 9.14524, 9.15344, 8.76086, 8.82546, 8.85801, 8.78594, 8.83766, 8.7627, 8.8693, 8.76505, 8.95513, 8.94138, 8.60415, 8.49526, 8.5414, 8.6052, 8.49378, 8.54563, 8.69589, 8.47931, 8.31047, 8.34191, 8.33761, 8.38482, 8.03117, 8.21698, 8.01005, 8.36597, 8.35171, 8.1238, 8.08903, 8.03892, 7.85884, 7.86204, 7.76178, 7.63785, 8.03256, 7.82491, 7.57767, 7.87018, 7.89663, 7.66576, 7.41891, 7.57945, 7.45949, 7.58407, 7.3365, 7.75478, 7.39312, 7.46005, 7.32601, 7.32261, 7.53324, 7.28432, 7.3906, 7.10455, 7.1031, 7.135, 7.2333, 6.91495, 7.07308, 7.17321, 7.08148, 6.95568, 6.83552, 7.07146, 7.13597, 6.77633, 6.6537, 6.79923, 6.81094, 6.80156, 6.80623, 6.72479, 6.46997, 6.7029, 6.67891, 6.50414, 6.69017, 6.80201, 6.66742, 6.78223, 6.74908, 6.68039, 6.55851, 6.65127, 6.45882, 6.71595, 6.3003, 6.29947, 6.35127, 6.43626, 6.39728, 6.5005, 6.33652, 6.38489, 6.2805, 6.24364, 6.44007, 6.36837, 6.36408, 6.20465, 6.19665, 6.27951, 6.42484, 6.24039, 6.18602, 6.21368, 6.14857, 6.09651, 6.10359, 6.28963, 6.44182, 6.28988, 6.33247, 6.13546, 6.21108, 6.0349, 6.06273, 5.987, 6.28025, 6.22641, 5.99808, 5.81837, 6.16027, 5.88364, 6.139, 5.82189, 6.19536, 6.17777, 6.11785, 5.96408, 6.14649, 5.9753, 6.22609, 5.92665, 5.82529, 5.80636, 5.7182, 6.04353, 6.02584, 6.092, 5.9119, 6.06757, 5.99273, 6.02669, 6.01523, 5.97662, 5.86429, 5.97653, 5.6431, 5.7275, 5.9135, 5.8664, 5.88797, 5.78842, 5.86055, 5.75215, 5.58542, 5.74699, 5.6532, 5.85871, 5.63063, 5.7325, 5.73883, 5.92312, 5.66992, 5.87123, 5.76346, 5.89613, 5.35339, 5.91985, 5.89554, 5.87623, 5.43362, 5.42829, 5.64744, 5.61678, 5.5103, 5.59917, 5.6988, 5.49854, 5.77013, 5.53314, 5.61954, 5.64553, 5.64008, 5.53513, 5.63528, 5.69717, 5.71522, 5.60874, 5.6802, 5.39435, 5.70021, 5.64782, 5.44435, 5.60824, 5.65007, 5.57098, 5.36362, 5.55798, 5.50433, 5.50082, 5.39457, 5.57452, 5.62082, 5.40855, 5.54177, 5.50319, 5.34993, 5.52256, 5.42475, 5.457, 5.33418, 5.08125, 5.49351, 5.58285, 5.72877, 5.42977, 5.613, 5.64847, 5.2484, 5.28756, 5.41008, 5.40961, 5.34061, 5.51276, 5.19903, 5.31256, 5.26266, 5.3907, 5.27539, 5.46188, 5.55243, 5.32608, 5.4523, 5.34935, 5.085, 5.3281, 5.26395, 5.31744, 5.12555, 5.28677, 5.2827, 5.486, 5.17172, 5.28031, 5.22155, 5.37027, 4.99359, 4.92973, 5.33403, 5.3997, 5.23719, 5.33061, 5.11473, 5.1717, 5.27268, 5.07733, 5.2767, 5.0858, 5.35129, 5.2583, 5.16657, 5.25468, 5.05243, 5.32453, 5.06278, 5.03705, 5.15134, 5.12068, 5.28265, 5.15883, 5.28883, 5.10618, 5.10727, 5.2621, 5.33107, 5.26622, 5.20237, 5.15543, 5.29779, 4.95636, 5.21799, 5.10164, 5.30924, 5.18679, 5.19599, 5.12317, 4.99367, 5.00306, 5.23171, 5.32198, 5.10695, 5.0647, 4.92646, 5.13309, 5.12718, 4.93681, 5.34691, 5.03142, 5.11047, 5.16889, 5.01087, 5.07032, 5.07588, 5.00122, 5.08773, 5.16951, 4.98692, 5.18998, 4.93899, 4.92741, 5.07395, 5.00085, 4.91692, 4.78186, 4.94917, 5.12365, 5.02541, 5.02437, 5.33759, 4.96582, 5.00145, 5.05138, 4.81301, 4.74456, 5.00203, 5.04679, 4.88367, 4.95882, 5.05212, 5.03024, 4.82289, 4.89705, 4.91162, 4.83722, 4.75468, 5.01694, 4.75625, 5.21634, 4.78922, 4.99899, 4.74083, 4.79117, 4.82499, 4.65555, 4.66118, 4.84502, 4.812, 4.80818, 4.93087, 4.88819, 4.92996, 4.77146, 4.88927, 4.73848, 4.91779, 4.96467, 4.87947, 4.7104, 4.78793, 4.90438, 4.71479, 4.86815, 4.69617, 4.69095, 4.65249]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4294967296.0, 134217728.0, 4194304.0, 131072.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4294967296.0, 134217728.0, 4194304.0, 131072.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 65536.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0, 131072.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95639, 179.95639, 179.95639, 179.95639, 179.95639, 179.95639, 179.95636, 179.95616, 179.95595, 179.9552, 179.95465, 179.95432, 179.95352, 179.953, 179.95229, 179.95172, 179.95114, 179.95059, 179.95015, 179.94978, 179.94951, 179.94933, 179.94916, 179.94899, 179.94891, 179.94894, 179.94923, 179.95026, 179.95171, 179.9529, 179.95413, 179.95543, 179.95691, 179.95865, 179.96053, 179.96269, 179.96513, 179.96796, 179.97112, 179.97466, 179.97838, 179.98239, 179.98705, 179.9922, 179.99811, 180.00458, 180.01144, 180.0188, 180.0265, 180.0349, 180.04382, 180.05347, 180.06361, 180.07454, 180.0863, 180.09869, 180.1114, 180.12436, 180.13821, 180.15294, 180.16814, 180.18376, 180.20035, 180.21758, 180.23528, 180.25388, 180.27333, 180.2935, 180.31477, 180.33707, 180.36023, 180.38481, 180.4104, 180.43663, 180.46335, 180.49043, 180.51775, 180.54597, 180.57475, 180.60458, 180.63466, 180.66501, 180.69615, 180.72832, 180.76106, 180.79457, 180.82857, 180.86211, 180.89636, 180.93251, 180.97021, 181.00865, 181.04654, 181.08444, 181.12204, 181.1591, 181.19463, 181.22873, 181.26352, 181.29965, 181.33498, 181.36926, 181.40433, 181.44101, 181.47787, 181.51541, 181.55309, 181.58995, 181.62593, 181.66238, 181.69963, 181.73865, 181.77856, 181.819, 181.85893, 181.89955, 181.94034, 181.98015, 182.01802, 182.05594, 182.09499, 182.13466, 182.17516, 182.21599, 182.25551, 182.29494, 182.33302, 182.36942, 182.40552, 182.44077, 182.47746, 182.51506, 182.55521, 182.59557, 182.63631, 182.67693, 182.71771, 182.75752, 182.79524, 182.83229, 182.8694, 182.90648, 182.94411, 182.98082, 183.01617, 183.05077, 183.08421, 183.11528, 183.14688, 183.17844, 183.21207, 183.24745, 183.28352, 183.31885, 183.35526, 183.39171, 183.42731, 183.46333, 183.49973, 183.53497, 183.57001, 183.60588, 183.64211, 183.6795, 183.71835, 183.75874, 183.79941, 183.83905, 183.87886, 183.91798, 183.95557, 183.99252, 184.02957, 184.06734, 184.1066, 184.14734, 184.18813, 184.22699, 184.26306, 184.29767, 184.33336, 184.36948, 184.40587, 184.44305, 184.48088, 184.51953, 184.55611, 184.58971, 184.62381, 184.65984, 184.6958, 184.73257, 184.76843, 184.80443, 184.84024, 184.87787, 184.91624, 184.9561, 184.99586, 185.03816, 185.08003, 185.12041, 185.16002, 185.19998, 185.23941, 185.27916, 185.31915, 185.35942, 185.3989, 185.43639, 185.4734, 185.51125, 185.54845, 185.5865, 185.62511, 185.66444, 185.70372, 185.74438, 185.78564, 185.82716, 185.86717, 185.90334, 185.937, 185.97195, 186.00873, 186.04741, 186.0872, 186.12794, 186.16808, 186.20654, 186.24687, 186.28903, 186.3307, 186.3723, 186.4149, 186.45834, 186.50229, 186.54523, 186.58723, 186.62804, 186.66795, 186.70871, 186.75044, 186.79398, 186.83716, 186.88002, 186.92215, 186.96371, 187.00597, 187.04924, 187.09216, 187.13554, 187.17883, 187.22208, 187.26509, 187.30769, 187.34932, 187.39163, 187.43529, 187.47867, 187.52255, 187.5659, 187.6091, 187.65163, 187.6926, 187.7334, 187.77498, 187.81706, 187.85999, 187.90363, 187.94743, 187.99174, 188.03735, 188.08296, 188.12976, 188.17722, 188.22394, 188.27153, 188.31853, 188.3636, 188.40756, 188.45032, 188.49333, 188.53738, 188.58321, 188.62881, 188.67557, 188.722, 188.76859, 188.81543, 188.86082, 188.90515, 188.94725, 188.9901, 189.0343, 189.07765, 189.12099, 189.16522, 189.21011, 189.25642, 189.3047, 189.35202, 189.39963, 189.4478, 189.49484, 189.5425, 189.59079, 189.63968, 189.68971, 189.74034, 189.79134, 189.84206, 189.89209, 189.9409, 189.99072, 190.04274, 190.09349, 190.14539, 190.19702, 190.24873, 190.30104, 190.35287, 190.4046, 190.45503, 190.50591, 190.55637, 190.60674, 190.65721, 190.70746, 190.75826, 190.80876, 190.8571, 190.90599, 190.95639, 191.00842, 191.06157, 191.11446, 191.16747, 191.22209, 191.2751, 191.32907, 191.38292, 191.43571, 191.48894, 191.54314, 191.59731, 191.65059, 191.70349, 191.75555, 191.80978, 191.86417, 191.91782, 191.97105, 192.02429, 192.0775, 192.13084, 192.18512, 192.24028, 192.29506, 192.35016, 192.40651, 192.4622, 192.51747, 192.57224, 192.62952, 192.687, 192.74483, 192.80281, 192.86006, 192.91705, 192.97177, 193.02679, 193.08273, 193.13742, 193.1917, 193.24458, 193.29779, 193.35132, 193.40689, 193.46413, 193.52164, 193.57927, 193.63789, 193.69646, 193.75464, 193.81409, 193.87488, 193.93707, 193.99841, 194.05937, 194.11984, 194.17958, 194.23772, 194.29633, 194.35521, 194.41174, 194.46733, 194.52335, 194.58064, 194.6398]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95639, 179.95639, 179.95639, 179.95639, 179.95639, 179.95639, 179.95636, 179.95616, 179.95595, 179.9552, 179.95465, 179.95432, 179.95352, 179.953, 179.95229, 179.95172, 179.95114, 179.95059, 179.95015, 179.94978, 179.94951, 179.94933, 179.94916, 179.94899, 179.94891, 179.94894, 179.94923, 179.95026, 179.95171, 179.9529, 179.95413, 179.95543, 179.95691, 179.95865, 179.96053, 179.96269, 179.96513, 179.96796, 179.97112, 179.97466, 179.97838, 179.98239, 179.98705, 179.9922, 179.99811, 180.00458, 180.01144, 180.0188, 180.0265, 180.0349, 180.04382, 180.05347, 180.06361, 180.07454, 180.0863, 180.09869, 180.1114, 180.12436, 180.13821, 180.15294, 180.16814, 180.18376, 180.20035, 180.21758, 180.23528, 180.25388, 180.27333, 180.2935, 180.31477, 180.33707, 180.36023, 180.38481, 180.4104, 180.43663, 180.46335, 180.49043, 180.51775, 180.54597, 180.57475, 180.60458, 180.63466, 180.66501, 180.69615, 180.72832, 180.76106, 180.79457, 180.82857, 180.86211, 180.89636, 180.93251, 180.97021, 181.00865, 181.04654, 181.08444, 181.12204, 181.1591, 181.19463, 181.22873, 181.26352, 181.29965, 181.33498, 181.36926, 181.40433, 181.44101, 181.47787, 181.51541, 181.55309, 181.58995, 181.62593, 181.66238, 181.69963, 181.73865, 181.77856, 181.819, 181.85893, 181.89955, 181.94034, 181.98015, 182.01802, 182.05594, 182.09499, 182.13466, 182.17516, 182.21599, 182.25551, 182.29494, 182.33302, 182.36942, 182.40552, 182.44077, 182.47746, 182.51506, 182.55521, 182.59557, 182.63631, 182.67693, 182.71771, 182.75752, 182.79524, 182.83229, 182.8694, 182.90648, 182.94411, 182.98082, 183.01617, 183.05077, 183.08421, 183.11528, 183.14688, 183.17844, 183.21207, 183.24745, 183.28352, 183.31885, 183.35526, 183.39171, 183.42731, 183.46333, 183.49973, 183.53497, 183.57001, 183.60588, 183.64211, 183.6795, 183.71835, 183.75874, 183.79941, 183.83905, 183.87886, 183.91798, 183.95557, 183.99252, 184.02957, 184.06734, 184.1066, 184.14734, 184.18813, 184.22699, 184.26306, 184.29767, 184.33336, 184.36948, 184.40587, 184.44305, 184.48088, 184.51953, 184.55611, 184.58971, 184.62381, 184.65984, 184.6958, 184.73257, 184.76843, 184.80443, 184.84024, 184.87787, 184.91624, 184.9561, 184.99586, 185.03816, 185.08003, 185.12041, 185.16002, 185.19998, 185.23941, 185.27916, 185.31915, 185.35942, 185.3989, 185.43639, 185.4734, 185.51125, 185.54845, 185.5865, 185.62511, 185.66444, 185.70372, 185.74438, 185.78564, 185.82716, 185.86717, 185.90334, 185.937, 185.97195, 186.00873, 186.04741, 186.0872, 186.12794, 186.16808, 186.20654, 186.24687, 186.28903, 186.3307, 186.3723, 186.4149, 186.45834, 186.50229, 186.54523, 186.58723, 186.62804, 186.66795, 186.70871, 186.75044, 186.79398, 186.83716, 186.88002, 186.92215, 186.96371, 187.00597, 187.04924, 187.09216, 187.13554, 187.17883, 187.22208, 187.26509, 187.30769, 187.34932, 187.39163, 187.43529, 187.47867, 187.52255, 187.5659, 187.6091, 187.65163, 187.6926, 187.7334, 187.77498, 187.81706, 187.85999, 187.90363, 187.94743, 187.99174, 188.03735, 188.08296, 188.12976, 188.17722, 188.22394, 188.27153, 188.31853, 188.3636, 188.40756, 188.45032, 188.49333, 188.53738, 188.58321, 188.62881, 188.67557, 188.722, 188.76859, 188.81543, 188.86082, 188.90515, 188.94725, 188.9901, 189.0343, 189.07765, 189.12099, 189.16522, 189.21011, 189.25642, 189.3047, 189.35202, 189.39963, 189.4478, 189.49484, 189.5425, 189.59079, 189.63968, 189.68971, 189.74034, 189.79134, 189.84206, 189.89209, 189.9409, 189.99072, 190.04274, 190.09349, 190.14539, 190.19702, 190.24873, 190.30104, 190.35287, 190.4046, 190.45503, 190.50591, 190.55637, 190.60674, 190.65721, 190.70746, 190.75826, 190.80876, 190.8571, 190.90599, 190.95639, 191.00842, 191.06157, 191.11446, 191.16747, 191.22209, 191.2751, 191.32907, 191.38292, 191.43571, 191.48894, 191.54314, 191.59731, 191.65059, 191.70349, 191.75555, 191.80978, 191.86417, 191.91782, 191.97105, 192.02429, 192.0775, 192.13084, 192.18512, 192.24028, 192.29506, 192.35016, 192.40651, 192.4622, 192.51747, 192.57224, 192.62952, 192.687, 192.74483, 192.80281, 192.86006, 192.91705, 192.97177, 193.02679, 193.08273, 193.13742, 193.1917, 193.24458, 193.29779, 193.35132, 193.40689, 193.46413, 193.52164, 193.57927, 193.63789, 193.69646, 193.75464, 193.81409, 193.87488, 193.93707, 193.99841, 194.05937, 194.11984, 194.17958, 194.23772, 194.29633, 194.35521, 194.41174, 194.46733, 194.52335, 194.58064, 194.6398]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [18.07681, 0.38236, 0.3815, 0.38004, 0.39049, 0.39656, 0.39642, 0.39048, 0.39523, 0.39194, 0.5552, 0.3948, 0.39398, 0.39561, 0.39214, 0.39537, 0.39216, 0.39261, 0.39694, 0.39356, 0.4003, 0.39114, 0.39355, 0.3919, 0.39064, 0.40086, 0.39355, 0.39139, 0.38492, 0.3927, 0.40428, 0.38479, 0.38466, 0.38299, 0.38174, 0.38636, 0.38086, 0.38401, 0.38601, 0.40511, 0.38629, 0.38521, 0.3855, 0.38256, 0.38493, 0.38553, 0.38438, 0.38462, 0.38628, 0.38214, 0.38492, 0.38322, 0.38706, 0.38103, 0.38314, 0.38469, 0.38271, 0.38565, 0.38283, 0.38163, 0.37833, 0.38621, 0.37993, 0.37921, 0.38058, 0.38093, 0.38301, 0.38316, 0.38564, 0.38136, 0.38386, 0.38121, 0.38145, 0.37922, 0.48103, 0.37987, 0.38025, 0.38308, 0.38613, 0.38258, 0.38336, 0.38508, 0.3887, 0.38459, 0.38233, 0.38094, 0.38026, 0.38316, 0.3802, 0.38401, 0.38409, 0.38327, 0.39188, 0.38081, 0.38297, 0.38391, 0.38075, 0.38566, 0.38249, 0.38281, 0.38433, 0.38249, 0.37955, 0.38003, 0.47628, 0.38394, 0.38015, 0.40241, 0.37987, 0.38149, 0.38158, 0.38618, 0.38356, 0.38072, 0.3889, 0.38918, 0.38574, 0.38775, 0.38338, 0.39021, 0.38146, 0.38236, 0.38742, 0.3868, 0.38407, 0.38593, 0.38727, 0.39089, 0.39337, 0.38585, 0.38443, 0.38667, 0.3868, 0.39023, 0.49507, 0.38161, 0.38081, 0.38199, 0.48238, 0.53269, 0.38537, 0.38444, 0.38705, 0.39224, 0.38871, 0.3845, 0.38286, 0.38071, 0.38022, 0.38228, 0.38177, 0.38417, 0.3801, 0.38435, 0.38639, 0.38626, 0.38489, 0.38587, 0.38488, 0.38407, 0.3867, 0.38401, 0.3866, 0.38593, 0.38916, 0.3833, 0.38389, 0.3843, 0.38359, 0.38697, 0.38383, 0.38577, 0.38399, 0.38402, 0.38788, 0.3861, 0.38511, 0.38672, 0.38227, 0.38915, 0.38446, 0.3859, 0.37898, 0.381, 0.38613, 0.38362, 0.3831, 0.37854, 0.37897, 0.37818, 0.37983, 0.38369, 0.37982, 0.38105, 0.38549, 0.38522, 0.38518, 0.38435, 0.47441, 0.38233, 0.37927, 0.38248, 0.38035, 0.37886, 0.38094, 0.3816, 0.38623, 0.38907, 0.38824, 0.38363, 0.38085, 0.38241, 0.38688, 0.3809, 0.38401, 0.3846, 0.38278, 0.38686, 0.38509, 0.38569, 0.38138, 0.38221, 0.38366, 0.39376, 0.39173, 0.38031, 0.38231, 0.47746, 0.38191, 0.38528, 0.38919, 0.38627, 0.38485, 0.39016, 0.48709, 0.39134, 0.38991, 0.38575, 0.3826, 0.38101, 0.38387, 0.38025, 0.37997, 0.50302, 0.38436, 0.38473, 0.38639, 0.38633, 0.3928, 0.38343, 0.38522, 0.38229, 0.37817, 0.38096, 0.38116, 0.3867, 0.38377, 0.38146, 0.38226, 0.38398, 0.39339, 0.3803, 0.48334, 0.38398, 0.38072, 0.38756, 0.38406, 0.38475, 0.3865, 0.3837, 0.39344, 0.38796, 0.38926, 0.38703, 0.38603, 0.37954, 0.38341, 0.38785, 0.38335, 0.38263, 0.38197, 0.38334, 0.3861, 0.38808, 0.38389, 0.38779, 0.39044, 0.38432, 0.38303, 0.38348, 0.38756, 0.38699, 0.47757, 0.38391, 0.38223, 0.38479, 0.38831, 0.38749, 0.384, 0.3864, 0.38554, 0.38656, 0.38469, 0.38559, 0.38552, 0.38634, 0.39068, 0.38718, 0.38906, 0.38314, 0.38526, 0.39355, 0.38547, 0.3918, 0.38838, 0.39149, 0.38788, 0.38735, 0.38776, 0.38498, 0.3845, 0.3809, 0.38438, 0.38342, 0.38109, 0.38385, 0.3847, 0.38354, 0.38456, 0.48679, 0.38819, 0.38623, 0.3908, 0.39049, 0.38764, 0.39009, 0.3899, 0.39171, 0.39325, 0.39116, 0.38744, 0.38994, 0.3945, 0.38791, 0.3872, 0.3882, 0.38525, 0.38534, 0.38602, 0.38534, 0.38256, 0.38598, 0.38572, 0.37898, 0.38512, 0.38512, 0.38361, 0.39213, 0.38551, 0.38269, 0.38516, 0.38696, 0.38679, 0.37971, 0.38365, 0.38484, 0.38698, 0.39395, 0.38701, 0.38655, 0.38288, 0.38233, 0.38642, 0.38468, 0.38309, 0.38362, 0.38617, 0.3863, 0.38907, 0.38471, 0.38686, 0.38576, 0.3853, 0.38783, 0.3863, 0.38804, 0.38654, 0.48838, 0.39169, 0.38856, 0.47555, 0.38859, 0.39202, 0.38824, 0.59598, 0.38895, 0.38921, 0.38633, 0.38705, 0.38574]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [0.02457, 0.00089, 0.00088, 0.00089, 0.00088, 0.00089, 0.00089, 0.00089, 0.0009, 0.00089, 0.00091, 0.00095, 0.00088, 0.0009, 0.00088, 0.00088, 0.00089, 0.0009, 0.0009, 0.00089, 0.0009, 0.00088, 0.00088, 0.00088, 0.00089, 0.00089, 0.00089, 0.00088, 0.00087, 0.00088, 0.00088, 0.00088, 0.00088, 0.00089, 0.00093, 0.00088, 0.00088, 0.0009, 0.00092, 0.00089, 0.00088, 0.00088, 0.00089, 0.00088, 0.00089, 0.00089, 0.00089, 0.00099, 0.00088, 0.00088, 0.00089, 0.00089, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.0009, 0.00126, 0.00088, 0.00088, 0.00088, 0.00094, 0.00088, 0.00087, 0.00088, 0.00087, 0.00088, 0.00088, 0.0009, 0.00087, 0.00088, 0.00088, 0.00088, 0.00087, 0.00088, 0.00087, 0.00125, 0.00093, 0.0009, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.00098, 0.00088, 0.00112, 0.00088, 0.00088, 0.00089, 0.00087, 0.00088, 0.00087, 0.00088, 0.00088, 0.00088, 0.00089, 0.0009, 0.00087, 0.00088, 0.00088, 0.00091, 0.00088, 0.00088, 0.00088, 0.00088, 0.00092, 0.00087, 0.00066, 0.00088, 0.00088, 0.0009, 0.00065, 0.00088, 0.00088, 0.00066, 0.00089, 0.00089, 0.00066, 0.00088, 0.001, 0.00088, 0.00088, 0.0009, 0.00066, 0.00066, 0.00088, 0.00067, 0.00089, 0.00089, 0.00067, 0.00088, 0.00089, 0.00087, 0.00087, 0.00095, 0.00088, 0.00087, 0.00088, 0.00087, 0.00089, 0.00089, 0.00088, 0.00089, 0.00089, 0.00088, 0.00089, 0.0009, 0.00087, 0.00087, 0.00089, 0.00088, 0.00087, 0.00087, 0.00087, 0.00087, 0.00088, 0.00088, 0.00089, 0.00088, 0.0009, 0.00089, 0.00087, 0.00087, 0.00087, 0.00089, 0.00089, 0.00094, 0.00088, 0.00087, 0.00087, 0.00088, 0.00088, 0.00087, 0.00087, 0.00088, 0.00088, 0.00088, 0.00087, 0.00087, 0.00087, 0.00087, 0.00088, 0.00088, 0.00087, 0.00087, 0.00098, 0.00088, 0.00091, 0.00087, 0.00087, 0.00089, 0.00088, 0.00088, 0.00088, 0.00091, 0.00087, 0.00088, 0.00107, 0.00095, 0.00088, 0.00087, 0.00088, 0.00094, 0.00093, 0.00087, 0.00089, 0.00087, 0.00088, 0.00087, 0.00089, 0.00087, 0.00087, 0.00087, 0.00087, 0.00088, 0.00089, 0.00087, 0.00087, 0.00088, 0.00089, 0.00087, 0.00087, 0.00094, 0.00088, 0.00087, 0.00089, 0.00093, 0.00088, 0.00087, 0.00087, 0.00088, 0.00088, 0.00088, 0.00088, 0.00095, 0.00087, 0.00087, 0.00087, 0.00087, 0.00087, 0.00108, 0.00087, 0.00089, 0.00089, 0.00089, 0.00088, 0.001, 0.00088, 0.00094, 0.00088, 0.00087, 0.00088, 0.00095, 0.0009, 0.00089, 0.00089, 0.00088, 0.00088, 0.00089, 0.00088, 0.0009, 0.00089, 0.00088, 0.00088, 0.00087, 0.00088, 0.00089, 0.00088, 0.00087, 0.00088, 0.00087, 0.00089, 0.00091, 0.00088, 0.00096, 0.00088, 0.00092, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.00088, 0.00087, 0.00089, 0.00088, 0.00091, 0.00095, 0.00088, 0.00088, 0.00095, 0.0009, 0.00089, 0.00092, 0.00093, 0.00099, 0.00088, 0.0009, 0.00087, 0.00088, 0.00096, 0.00088, 0.00097, 0.00087, 0.00088, 0.00087, 0.00088, 0.00088, 0.00098, 0.00089, 0.00097, 0.00087, 0.00087, 0.00087, 0.00088, 0.00089, 0.00088, 0.00089, 0.00088, 0.00088, 0.00087, 0.00087, 0.00099, 0.00089, 0.00088, 0.00088, 0.00087, 0.00088, 0.00088, 0.00089, 0.00087, 0.00088, 0.00088, 0.0009, 0.00091, 0.00089, 0.00087, 0.00088, 0.00089, 0.00089, 0.00087, 0.00088, 0.00094, 0.00088, 0.00088, 0.00088, 0.00088, 0.00089, 0.00087, 0.00106, 0.0009, 0.00089, 0.00088, 0.00096, 0.00089, 0.00098, 0.00088, 0.00088, 0.00088, 0.00091, 0.00087, 0.00089, 0.00088, 0.00088, 0.00088, 0.00088, 0.00087, 0.00089, 0.00089, 0.00088, 0.00089, 0.00089, 0.00088, 0.00091, 0.00089, 0.00087, 0.0009, 0.00088, 0.00089, 0.00088, 0.00093, 0.00116, 0.00101, 0.00088, 0.00095, 0.00092, 0.00089, 0.00088, 0.00087, 0.00089, 0.00105, 0.0009, 0.00087]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [0.01277, 0.00497, 0.00488, 0.00489, 0.00489, 0.00494, 0.00489, 0.0049, 0.00489, 0.00488, 0.00497, 0.00521, 0.0049, 0.00492, 0.00492, 0.0049, 0.00494, 0.00492, 0.00489, 0.00489, 0.00493, 0.0049, 0.00492, 0.0051, 0.00487, 0.00629, 0.005, 0.0049, 0.00492, 0.0049, 0.0049, 0.0049, 0.00488, 0.00492, 0.00535, 0.0049, 0.0049, 0.00494, 0.0049, 0.00494, 0.00489, 0.00489, 0.0049, 0.00491, 0.00492, 0.00491, 0.00599, 0.00523, 0.00489, 0.00489, 0.00491, 0.00491, 0.00491, 0.00494, 0.0049, 0.00489, 0.00491, 0.0049, 0.00491, 0.0049, 0.00491, 0.0049, 0.00525, 0.00492, 0.00493, 0.00489, 0.00489, 0.00492, 0.00491, 0.0049, 0.00491, 0.00491, 0.00492, 0.00489, 0.00489, 0.00493, 0.00493, 0.00498, 0.00519, 0.00491, 0.00491, 0.00492, 0.00498, 0.00492, 0.00494, 0.0049, 0.00489, 0.00567, 0.00489, 0.00491, 0.00491, 0.00524, 0.00489, 0.00491, 0.00489, 0.00504, 0.0056, 0.00501, 0.00491, 0.00493, 0.00492, 0.00491, 0.00491, 0.00491, 0.00489, 0.0049, 0.0049, 0.0049, 0.00492, 0.0049, 0.00491, 0.00491, 0.00602, 0.0049, 0.00494, 0.00489, 0.0049, 0.0049, 0.00491, 0.00492, 0.0049, 0.0049, 0.00491, 0.00598, 0.00492, 0.00491, 0.00489, 0.00494, 0.00491, 0.00491, 0.0049, 0.00494, 0.00492, 0.00544, 0.00488, 0.00491, 0.0049, 0.0049, 0.00503, 0.00491, 0.00491, 0.00491, 0.00493, 0.00494, 0.00493, 0.00492, 0.0049, 0.00492, 0.00488, 0.00489, 0.00515, 0.0049, 0.00498, 0.00492, 0.00493, 0.0049, 0.00491, 0.005, 0.00491, 0.00491, 0.00491, 0.00491, 0.00489, 0.00491, 0.0049, 0.0049, 0.00496, 0.00492, 0.00488, 0.00492, 0.00538, 0.00492, 0.00491, 0.00492, 0.00567, 0.00488, 0.00491, 0.00493, 0.00492, 0.00487, 0.00493, 0.0049, 0.00488, 0.00491, 0.00492, 0.0049, 0.00492, 0.0049, 0.0049, 0.00492, 0.0049, 0.0051, 0.0049, 0.00519, 0.00491, 0.00491, 0.00488, 0.00488, 0.00489, 0.00489, 0.00491, 0.00583, 0.0049, 0.0049, 0.00489, 0.00488, 0.0049, 0.00489, 0.00491, 0.00488, 0.0049, 0.00501, 0.00492, 0.00491, 0.0049, 0.0049, 0.0049, 0.00488, 0.0049, 0.00489, 0.00489, 0.0049, 0.00489, 0.00492, 0.00493, 0.00488, 0.0049, 0.00489, 0.0049, 0.00489, 0.00494, 0.00489, 0.00491, 0.00489, 0.00489, 0.0049, 0.00492, 0.00487, 0.00491, 0.00491, 0.00489, 0.00489, 0.00489, 0.00491, 0.00578, 0.0049, 0.00488, 0.00487, 0.00492, 0.0049, 0.00491, 0.00489, 0.00489, 0.00488, 0.0049, 0.00489, 0.00489, 0.00491, 0.00515, 0.00494, 0.0049, 0.00489, 0.00492, 0.00489, 0.00502, 0.00489, 0.00493, 0.00489, 0.00491, 0.00491, 0.00489, 0.0049, 0.00582, 0.00487, 0.00489, 0.0049, 0.00491, 0.00488, 0.00489, 0.00492, 0.00488, 0.00489, 0.00491, 0.00489, 0.00489, 0.0049, 0.00489, 0.00558, 0.00491, 0.0056, 0.00495, 0.00488, 0.00491, 0.00489, 0.00489, 0.00488, 0.0049, 0.0049, 0.00489, 0.00492, 0.00491, 0.0049, 0.00491, 0.00489, 0.0049, 0.00491, 0.00492, 0.00512, 0.00493, 0.00491, 0.00491, 0.0049, 0.00491, 0.00492, 0.00579, 0.00626, 0.00489, 0.00489, 0.0049, 0.00489, 0.00491, 0.00494, 0.00489, 0.00491, 0.0049, 0.0049, 0.00491, 0.00512, 0.0051, 0.00514, 0.00513, 0.00513, 0.00514, 0.00513, 0.00512, 0.00511, 0.00512, 0.00514, 0.0052, 0.00512, 0.00511, 0.00513, 0.00514, 0.00511, 0.00511, 0.00514, 0.00564, 0.00511, 0.00512, 0.00509, 0.00512, 0.00512, 0.00536, 0.00513, 0.00512, 0.00513, 0.00512, 0.00513, 0.00512, 0.00512, 0.00512, 0.00512, 0.00509, 0.00512, 0.00512, 0.00513, 0.00512, 0.00514, 0.00515, 0.00514, 0.00516, 0.00512, 0.00513, 0.00514, 0.00511, 0.00513, 0.00524, 0.00511, 0.00514, 0.00512, 0.00511, 0.00509, 0.00513, 0.00511, 0.00514, 0.00513, 0.00513, 0.00512, 0.0055, 0.0054, 0.00513, 0.0051, 0.0051, 0.00512, 0.00514, 0.00515, 0.00515]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [0.00686, 0.00099, 0.00098, 0.00098, 0.00098, 0.001, 0.00099, 0.00099, 0.00098, 0.00099, 0.00101, 0.00098, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.001, 0.001, 0.001, 0.00098, 0.00097, 0.00099, 0.00098, 0.00124, 0.00098, 0.00098, 0.00098, 0.00098, 0.00098, 0.00101, 0.00101, 0.001, 0.001, 0.00098, 0.00099, 0.001, 0.00102, 0.00098, 0.00098, 0.00099, 0.00098, 0.00098, 0.00098, 0.00097, 0.001, 0.00102, 0.00097, 0.00098, 0.00099, 0.001, 0.00097, 0.00102, 0.00099, 0.00098, 0.00098, 0.00098, 0.001, 0.001, 0.001, 0.00098, 0.00098, 0.00098, 0.00098, 0.00098, 0.00097, 0.00097, 0.00099, 0.00098, 0.00098, 0.00098, 0.00104, 0.00097, 0.00098, 0.00099, 0.00098, 0.00117, 0.00101, 0.00101, 0.00099, 0.00097, 0.00098, 0.00097, 0.00099, 0.00098, 0.00098, 0.00101, 0.00099, 0.00098, 0.00098, 0.00098, 0.001, 0.00097, 0.00097, 0.00098, 0.001, 0.00097, 0.00097, 0.00098, 0.00099, 0.00098, 0.00098, 0.00098, 0.00098, 0.00097, 0.00097, 0.00098, 0.001, 0.00099, 0.00097, 0.00098, 0.001, 0.00098, 0.00098, 0.00099, 0.00098, 0.00098, 0.00099, 0.00099, 0.00099, 0.00097, 0.00097, 0.00099, 0.00098, 0.00097, 0.001, 0.00099, 0.00098, 0.00099, 0.001, 0.00097, 0.00099, 0.00102, 0.00099, 0.00098, 0.00097, 0.00099, 0.00099, 0.001, 0.00097, 0.00097, 0.00098, 0.00099, 0.001, 0.001, 0.00098, 0.001, 0.001, 0.00097, 0.00101, 0.00097, 0.00099, 0.00099, 0.00098, 0.001, 0.00099, 0.00098, 0.001, 0.00097, 0.00098, 0.001, 0.00099, 0.00099, 0.00099, 0.00098, 0.00098, 0.00097, 0.00098, 0.00099, 0.00098, 0.00099, 0.00097, 0.00098, 0.00103, 0.00097, 0.00097, 0.001, 0.00099, 0.00098, 0.00098, 0.00099, 0.00097, 0.00098, 0.00098, 0.00101, 0.001, 0.00099, 0.00098, 0.00098, 0.00097, 0.00102, 0.00098, 0.00097, 0.00098, 0.00097, 0.00097, 0.00099, 0.00102, 0.00096, 0.00099, 0.00097, 0.00096, 0.00097, 0.00097, 0.00099, 0.00096, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00098, 0.00097, 0.00098, 0.00156, 0.00097, 0.00096, 0.00097, 0.00096, 0.001, 0.00101, 0.00097, 0.00099, 0.00097, 0.00096, 0.00098, 0.00098, 0.00103, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00099, 0.00097, 0.00096, 0.00098, 0.00098, 0.00097, 0.00098, 0.00099, 0.00099, 0.00098, 0.00097, 0.00098, 0.00097, 0.00098, 0.00099, 0.001, 0.00099, 0.00098, 0.001, 0.00099, 0.00099, 0.00101, 0.00102, 0.00099, 0.00099, 0.00098, 0.00098, 0.00099, 0.00098, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00098, 0.00101, 0.00099, 0.00099, 0.00099, 0.00097, 0.00099, 0.00099, 0.00098, 0.00098, 0.00104, 0.00098, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00097, 0.00099, 0.00098, 0.00098, 0.001, 0.00099, 0.00099, 0.00098, 0.00099, 0.00098, 0.00097, 0.00098, 0.00099, 0.00099, 0.00099, 0.00098, 0.00104, 0.00099, 0.00098, 0.00099, 0.00099, 0.001, 0.00099, 0.00098, 0.001, 0.00099, 0.00096, 0.00098, 0.00099, 0.00099, 0.001, 0.00099, 0.00097, 0.00099, 0.00099, 0.00098, 0.00099, 0.00099, 0.00103, 0.00099, 0.00098, 0.00099, 0.00097, 0.00098, 0.00099, 0.00098, 0.00098, 0.00101, 0.00098, 0.00099, 0.00099, 0.00098, 0.00156, 0.00103, 0.00098, 0.001, 0.00098, 0.00099, 0.00098, 0.00098, 0.00099, 0.00098, 0.001, 0.001, 0.00098, 0.00102, 0.00098, 0.00098, 0.00099, 0.00098, 0.00098, 0.00099, 0.001, 0.00098, 0.00098, 0.00098, 0.00098, 0.00098, 0.00099, 0.00097, 0.00099, 0.00096, 0.00102, 0.00098, 0.00099, 0.001, 0.001, 0.001, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.001, 0.001, 0.00104, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.001, 0.00099, 0.00099]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [0.00107, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00103, 0.00103, 0.00103, 0.00104, 0.00101, 0.00103, 0.00103, 0.00104, 0.00105, 0.00103, 0.00103, 0.00104, 0.00103, 0.00102, 0.00104, 0.00102, 0.00163, 0.00103, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00104, 0.00104, 0.00103, 0.00102, 0.00103, 0.00104, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00103, 0.00102, 0.00108, 0.00106, 0.00102, 0.00103, 0.00103, 0.00104, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00103, 0.00103, 0.00103, 0.00104, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00115, 0.00105, 0.00126, 0.00103, 0.00103, 0.00102, 0.00103, 0.00103, 0.00103, 0.00102, 0.00106, 0.00102, 0.00103, 0.00102, 0.00114, 0.00102, 0.00103, 0.00102, 0.00102, 0.00104, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00107, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00103, 0.00102, 0.00109, 0.00103, 0.00103, 0.00103, 0.00105, 0.00102, 0.00103, 0.00103, 0.00103, 0.00103, 0.00105, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00104, 0.00103, 0.00103, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00103, 0.00103, 0.00103, 0.00103, 0.00102, 0.00102, 0.00102, 0.00104, 0.00102, 0.00103, 0.00102, 0.00102, 0.00108, 0.00103, 0.00102, 0.00103, 0.00115, 0.00102, 0.00103, 0.00103, 0.00102, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00104, 0.00103, 0.00102, 0.00106, 0.00102, 0.00102, 0.00103, 0.00103, 0.00099, 0.001, 0.00103, 0.001, 0.001, 0.00105, 0.00101, 0.00099, 0.00099, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.00111, 0.001, 0.00099, 0.001, 0.00099, 0.00105, 0.00099, 0.00099, 0.001, 0.00099, 0.00099, 0.00099, 0.00099, 0.001, 0.001, 0.00099, 0.001, 0.00099, 0.00099, 0.00101, 0.00099, 0.00101, 0.001, 0.00099, 0.001, 0.00106, 0.001, 0.001, 0.001, 0.00104, 0.001, 0.001, 0.001, 0.00099, 0.00106, 0.001, 0.001, 0.001, 0.001, 0.00101, 0.00102, 0.00099, 0.00101, 0.00101, 0.001, 0.00099, 0.001, 0.00101, 0.00101, 0.00101, 0.00106, 0.001, 0.00101, 0.001, 0.00102, 0.001, 0.00101, 0.00106, 0.001, 0.001, 0.00101, 0.00099, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.00101, 0.00105, 0.00101, 0.00103, 0.00101, 0.001, 0.001, 0.00101, 0.00107, 0.001, 0.00106, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.00101, 0.00102, 0.00102, 0.001, 0.001, 0.00101, 0.001, 0.001, 0.001, 0.00106, 0.00107, 0.00099, 0.00107, 0.001, 0.00101, 0.001, 0.001, 0.001, 0.00101, 0.001, 0.001, 0.00101, 0.001, 0.00099, 0.001, 0.00101, 0.001, 0.001, 0.001, 0.001, 0.00107, 0.001, 0.001, 0.00101, 0.001, 0.001, 0.001, 0.00101, 0.00106, 0.00099, 0.00102, 0.00102, 0.001, 0.001, 0.001, 0.001, 0.00101, 0.00099, 0.00103, 0.00101, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.00101, 0.00103, 0.00102, 0.001, 0.00101, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.00099, 0.00102, 0.001, 0.001, 0.001, 0.00101, 0.00101, 0.001, 0.00099, 0.001, 0.00101, 0.00102, 0.00101, 0.00102, 0.00101, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.001, 0.001, 0.00101, 0.001, 0.001]}, "grad-norm": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [11.77525, 12.26804, 11.19281, 14.50237, 14.014, 11.57186, 8.3922, 7.10897, 4.47266, 4.00434, 3.4, 2.71736, 2.45629, 2.30739, 2.29493, 2.25132, 2.01839, 2.41173, 2.01298, 2.00525, 2.18932, 1.91353, 1.88951, 2.28883, 2.07903, 1.8844, 1.87495, 2.08513, 2.01874, 2.01118, 2.0102, 1.89229, 1.99489, 1.65446, 2.02134, 1.98456, 2.13312, 2.05074, 1.91832, 1.88506, 1.86975, 1.90714, 2.10548, 1.83107, 1.85561, 1.89757, 1.77389, 1.83901, 1.60882, 1.67073, 1.57953, 1.73056, 1.77582, 1.85094, 1.58796, 1.69243, 2.01012, 1.72305, 1.68342, 1.77634, 1.52051, 1.58604, 1.75613, 1.50876, 1.38814, 1.4853, 1.45829, 1.51675, 1.54655, 1.47158, 1.51099, 1.4708, 1.47268, 1.47452, 1.44323, 1.32185, 1.33599, 1.35564, 1.29533, 1.27928, 1.44962, 1.33226, 1.18991, 1.39956, 1.21257, 1.16175, 1.05645, 1.15134, 1.32979, 1.15427, 1.22191, 1.18197, 1.5911, 1.3589, 1.27604, 1.13871, 1.30626, 1.67866, 1.52014, 1.03431, 1.05476, 1.3049, 1.25479, 1.22714, 1.69201, 1.08131, 1.00908, 1.10419, 1.08066, 1.12768, 1.24403, 0.87723, 0.92972, 1.02293, 1.07062, 0.98243, 1.24502, 1.2897, 0.94461, 1.09023, 1.04658, 0.90251, 1.12421, 1.65432, 1.09595, 1.17882, 1.36022, 0.96059, 0.98043, 1.05339, 0.96416, 1.13229, 1.12844, 0.93359, 1.82877, 1.40011, 1.43068, 1.3027, 1.089, 1.64716, 1.37833, 1.56985, 1.16612, 1.85125, 1.24379, 1.71309, 1.39309, 1.27937, 1.17708, 1.73543, 1.05896, 1.24373, 1.38937, 1.36918, 1.42323, 1.77943, 1.13157, 1.27948, 1.19267, 1.34154, 1.40098, 1.16252, 1.42404, 1.2011, 1.00676, 1.48416, 1.13391, 1.33486, 1.5395, 1.27609, 1.42471, 1.30575, 1.22047, 1.81347, 1.74187, 1.56562, 1.47675, 1.51655, 1.70821, 1.44154, 1.50096, 1.28826, 1.74901, 1.90029, 1.42234, 1.44455, 1.76719, 1.84971, 1.73982, 1.24814, 1.53885, 1.39306, 1.62267, 1.27091, 1.59048, 1.06674, 1.40639, 1.29128, 1.69617, 1.31246, 1.4525, 1.29959, 1.38347, 1.4963, 1.45118, 1.62261, 1.8211, 1.48622, 1.35396, 1.364, 1.22302, 1.21036, 1.59732, 1.16621, 1.43458, 1.39264, 1.50491, 1.74865, 1.69988, 1.54719, 1.66156, 1.38606, 1.43929, 1.37822, 1.30248, 1.79296, 1.45361, 1.24972, 1.59221, 1.3686, 1.22551, 1.4158, 1.49894, 1.55813, 1.52684, 1.44435, 2.05338, 1.36019, 1.34284, 1.20815, 1.7307, 1.50669, 2.1527, 1.33714, 1.40114, 1.51052, 1.35152, 1.43159, 1.42052, 1.44093, 1.62874, 1.70468, 1.84621, 1.36339, 1.49409, 1.99351, 1.25437, 1.69787, 1.77453, 1.53971, 1.98798, 1.46692, 1.21412, 1.35855, 1.61255, 1.37129, 1.69078, 1.53059, 1.31087, 1.87886, 1.31042, 1.42235, 1.38194, 1.39636, 1.83392, 1.47651, 1.46996, 1.64541, 1.53153, 1.47267, 1.75528, 1.44853, 1.39865, 1.75941, 1.63286, 1.32552, 1.6715, 2.26149, 1.61139, 1.35216, 1.34936, 1.25166, 1.69472, 1.58245, 1.4379, 1.43627, 1.60457, 1.82215, 1.39138, 1.38678, 1.55708, 1.41296, 1.29816, 1.46066, 1.39994, 1.45437, 1.25759, 1.34921, 1.47682, 1.55246, 1.48338, 1.2271, 1.36154, 1.44453, 1.47772, 1.43402, 1.21249, 1.8034, 1.50506, 1.3131, 1.37503, 1.35584, 1.41307, 1.45748, 1.26629, 1.31721, 1.47686, 1.80237, 1.55348, 1.5369, 1.32871, 1.35524, 1.76226, 1.27945, 1.40786, 1.56063, 1.18102, 1.26595, 1.41714, 1.27185, 1.59955, 1.53902, 1.50856, 1.38342, 1.3716, 1.52597, 1.55924, 1.33891, 1.44137, 1.66178, 1.44058, 1.53213, 1.34923, 1.54826, 1.51369, 1.26166, 1.22057, 1.64988, 1.4183, 1.45977, 1.27097, 1.31805, 1.24715, 1.52412, 1.48112, 1.51313, 1.58975, 1.42731, 1.32647, 1.44532, 1.53827, 1.72661, 1.53155, 1.57687, 1.2723, 1.26403, 1.36125, 1.36611, 1.46818, 1.38679, 1.58433, 1.49566, 1.44288, 1.37271, 1.45317, 1.36918, 1.35342, 1.27732, 1.37088, 1.29411, 1.25869, 1.46478, 1.43992, 1.66108, 1.34488, 1.17599, 1.3251]}, "grad-norm vs samples": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [11.77525, 12.26804, 11.19281, 14.50237, 14.014, 11.57186, 8.3922, 7.10897, 4.47266, 4.00434, 3.4, 2.71736, 2.45629, 2.30739, 2.29493, 2.25132, 2.01839, 2.41173, 2.01298, 2.00525, 2.18932, 1.91353, 1.88951, 2.28883, 2.07903, 1.8844, 1.87495, 2.08513, 2.01874, 2.01118, 2.0102, 1.89229, 1.99489, 1.65446, 2.02134, 1.98456, 2.13312, 2.05074, 1.91832, 1.88506, 1.86975, 1.90714, 2.10548, 1.83107, 1.85561, 1.89757, 1.77389, 1.83901, 1.60882, 1.67073, 1.57953, 1.73056, 1.77582, 1.85094, 1.58796, 1.69243, 2.01012, 1.72305, 1.68342, 1.77634, 1.52051, 1.58604, 1.75613, 1.50876, 1.38814, 1.4853, 1.45829, 1.51675, 1.54655, 1.47158, 1.51099, 1.4708, 1.47268, 1.47452, 1.44323, 1.32185, 1.33599, 1.35564, 1.29533, 1.27928, 1.44962, 1.33226, 1.18991, 1.39956, 1.21257, 1.16175, 1.05645, 1.15134, 1.32979, 1.15427, 1.22191, 1.18197, 1.5911, 1.3589, 1.27604, 1.13871, 1.30626, 1.67866, 1.52014, 1.03431, 1.05476, 1.3049, 1.25479, 1.22714, 1.69201, 1.08131, 1.00908, 1.10419, 1.08066, 1.12768, 1.24403, 0.87723, 0.92972, 1.02293, 1.07062, 0.98243, 1.24502, 1.2897, 0.94461, 1.09023, 1.04658, 0.90251, 1.12421, 1.65432, 1.09595, 1.17882, 1.36022, 0.96059, 0.98043, 1.05339, 0.96416, 1.13229, 1.12844, 0.93359, 1.82877, 1.40011, 1.43068, 1.3027, 1.089, 1.64716, 1.37833, 1.56985, 1.16612, 1.85125, 1.24379, 1.71309, 1.39309, 1.27937, 1.17708, 1.73543, 1.05896, 1.24373, 1.38937, 1.36918, 1.42323, 1.77943, 1.13157, 1.27948, 1.19267, 1.34154, 1.40098, 1.16252, 1.42404, 1.2011, 1.00676, 1.48416, 1.13391, 1.33486, 1.5395, 1.27609, 1.42471, 1.30575, 1.22047, 1.81347, 1.74187, 1.56562, 1.47675, 1.51655, 1.70821, 1.44154, 1.50096, 1.28826, 1.74901, 1.90029, 1.42234, 1.44455, 1.76719, 1.84971, 1.73982, 1.24814, 1.53885, 1.39306, 1.62267, 1.27091, 1.59048, 1.06674, 1.40639, 1.29128, 1.69617, 1.31246, 1.4525, 1.29959, 1.38347, 1.4963, 1.45118, 1.62261, 1.8211, 1.48622, 1.35396, 1.364, 1.22302, 1.21036, 1.59732, 1.16621, 1.43458, 1.39264, 1.50491, 1.74865, 1.69988, 1.54719, 1.66156, 1.38606, 1.43929, 1.37822, 1.30248, 1.79296, 1.45361, 1.24972, 1.59221, 1.3686, 1.22551, 1.4158, 1.49894, 1.55813, 1.52684, 1.44435, 2.05338, 1.36019, 1.34284, 1.20815, 1.7307, 1.50669, 2.1527, 1.33714, 1.40114, 1.51052, 1.35152, 1.43159, 1.42052, 1.44093, 1.62874, 1.70468, 1.84621, 1.36339, 1.49409, 1.99351, 1.25437, 1.69787, 1.77453, 1.53971, 1.98798, 1.46692, 1.21412, 1.35855, 1.61255, 1.37129, 1.69078, 1.53059, 1.31087, 1.87886, 1.31042, 1.42235, 1.38194, 1.39636, 1.83392, 1.47651, 1.46996, 1.64541, 1.53153, 1.47267, 1.75528, 1.44853, 1.39865, 1.75941, 1.63286, 1.32552, 1.6715, 2.26149, 1.61139, 1.35216, 1.34936, 1.25166, 1.69472, 1.58245, 1.4379, 1.43627, 1.60457, 1.82215, 1.39138, 1.38678, 1.55708, 1.41296, 1.29816, 1.46066, 1.39994, 1.45437, 1.25759, 1.34921, 1.47682, 1.55246, 1.48338, 1.2271, 1.36154, 1.44453, 1.47772, 1.43402, 1.21249, 1.8034, 1.50506, 1.3131, 1.37503, 1.35584, 1.41307, 1.45748, 1.26629, 1.31721, 1.47686, 1.80237, 1.55348, 1.5369, 1.32871, 1.35524, 1.76226, 1.27945, 1.40786, 1.56063, 1.18102, 1.26595, 1.41714, 1.27185, 1.59955, 1.53902, 1.50856, 1.38342, 1.3716, 1.52597, 1.55924, 1.33891, 1.44137, 1.66178, 1.44058, 1.53213, 1.34923, 1.54826, 1.51369, 1.26166, 1.22057, 1.64988, 1.4183, 1.45977, 1.27097, 1.31805, 1.24715, 1.52412, 1.48112, 1.51313, 1.58975, 1.42731, 1.32647, 1.44532, 1.53827, 1.72661, 1.53155, 1.57687, 1.2723, 1.26403, 1.36125, 1.36611, 1.46818, 1.38679, 1.58433, 1.49566, 1.44288, 1.37271, 1.45317, 1.36918, 1.35342, 1.27732, 1.37088, 1.29411, 1.25869, 1.46478, 1.43992, 1.66108, 1.34488, 1.17599, 1.3251]}, "num-zeros": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [951.0, 1294.0, 1060.0, 971.0, 901.0, 1117.0, 1205.0, 1364.0, 1468.0, 1319.0, 1539.0, 1911.0, 2180.0, 1576.0, 2216.0, 1925.0, 2038.0, 2028.0, 2476.0, 2015.0, 2201.0, 2215.0, 2438.0, 3135.0, 2444.0, 2806.0, 2540.0, 2188.0, 2052.0, 2885.0, 2408.0, 3553.0, 2417.0, 2497.0, 2486.0, 3667.0, 2116.0, 2243.0, 2127.0, 2649.0, 3818.0, 2985.0, 2311.0, 2810.0, 2580.0, 2214.0, 2672.0, 2502.0, 2376.0, 2941.0, 3128.0, 2507.0, 2600.0, 2152.0, 2790.0, 3240.0, 2769.0, 2720.0, 2392.0, 3522.0, 2236.0, 2883.0, 2397.0, 2586.0, 2219.0, 3154.0, 2799.0, 2803.0, 2345.0, 2563.0, 2171.0, 2874.0, 2837.0, 2656.0, 3389.0, 2526.0, 2817.0, 2625.0, 3000.0, 2814.0, 2754.0, 2414.0, 3081.0, 2380.0, 2876.0, 2737.0, 2780.0, 2271.0, 2333.0, 2839.0, 2519.0, 3210.0, 2404.0, 2291.0, 2433.0, 2383.0, 2435.0, 1919.0, 2351.0, 2585.0, 2779.0, 2221.0, 2014.0, 2114.0, 1881.0, 2304.0, 2397.0, 2309.0, 2239.0, 2116.0, 2239.0, 2377.0, 2323.0, 2496.0, 2298.0, 2773.0, 2696.0, 1952.0, 2435.0, 2042.0, 2813.0, 2452.0, 2068.0, 2032.0, 2127.0, 2176.0, 2056.0, 2569.0, 2495.0, 2156.0, 2202.0, 2372.0, 2368.0, 2313.0, 1956.0, 2287.0, 2471.0, 2251.0, 2132.0, 1626.0, 2076.0, 2288.0, 2009.0, 1987.0, 2433.0, 1651.0, 2033.0, 2061.0, 1927.0, 2837.0, 2589.0, 2063.0, 1738.0, 1964.0, 2334.0, 1899.0, 2516.0, 2136.0, 2214.0, 1965.0, 1875.0, 2415.0, 1921.0, 2352.0, 2174.0, 1887.0, 2165.0, 2616.0, 1911.0, 1825.0, 1959.0, 1908.0, 1822.0, 1574.0, 1545.0, 2160.0, 1942.0, 2081.0, 1733.0, 2008.0, 2010.0, 2212.0, 1875.0, 1390.0, 1972.0, 2540.0, 1825.0, 2152.0, 1632.0, 2232.0, 1792.0, 1887.0, 1971.0, 2046.0, 1779.0, 2139.0, 2024.0, 1999.0, 1614.0, 1985.0, 1902.0, 2128.0, 2445.0, 2671.0, 2214.0, 2029.0, 2081.0, 2209.0, 2226.0, 1957.0, 2210.0, 2419.0, 2685.0, 2294.0, 1932.0, 2118.0, 1963.0, 1818.0, 1841.0, 2149.0, 2110.0, 2155.0, 1868.0, 2220.0, 2120.0, 2379.0, 1886.0, 2361.0, 1763.0, 2055.0, 1972.0, 2155.0, 1934.0, 2167.0, 1959.0, 1882.0, 1705.0, 1826.0, 1964.0, 2224.0, 1818.0, 1883.0, 1743.0, 2488.0, 2393.0, 2103.0, 2005.0, 2728.0, 2142.0, 2054.0, 1951.0, 1819.0, 2038.0, 2170.0, 2265.0, 1808.0, 2431.0, 1807.0, 2184.0, 2053.0, 1687.0, 1931.0, 2549.0, 2587.0, 1986.0, 2273.0, 2103.0, 2063.0, 2204.0, 2021.0, 2110.0, 2428.0, 2484.0, 2060.0, 2244.0, 2025.0, 1999.0, 1965.0, 1906.0, 2137.0, 2024.0, 2234.0, 1998.0, 2022.0, 1943.0, 2254.0, 2008.0, 1619.0, 1850.0, 2446.0, 2316.0, 1952.0, 2008.0, 2201.0, 2018.0, 2191.0, 1856.0, 2363.0, 2138.0, 2632.0, 1897.0, 2331.0, 1915.0, 2017.0, 2347.0, 2073.0, 2221.0, 2341.0, 1910.0, 1944.0, 2197.0, 2136.0, 2140.0, 2057.0, 2254.0, 1992.0, 2377.0, 1829.0, 2323.0, 2256.0, 2248.0, 2664.0, 2091.0, 2351.0, 2363.0, 2417.0, 1953.0, 2010.0, 2111.0, 2082.0, 2141.0, 2449.0, 2394.0, 2165.0, 2019.0, 2307.0, 2446.0, 2932.0, 2123.0, 2428.0, 2294.0, 2499.0, 2597.0, 2391.0, 2142.0, 2085.0, 2112.0, 2498.0, 2172.0, 2546.0, 2086.0, 2278.0, 2000.0, 2060.0, 2222.0, 2327.0, 2377.0, 2181.0, 1943.0, 2370.0, 2170.0, 2277.0, 2360.0, 2822.0, 2306.0, 2709.0, 2210.0, 2127.0, 2321.0, 2202.0, 2780.0, 2249.0, 2312.0, 2033.0, 2114.0, 2287.0, 2292.0, 2301.0, 2735.0, 2674.0, 2246.0, 2584.0, 2280.0, 2624.0, 2634.0, 2653.0, 2502.0, 2748.0, 2256.0, 2492.0, 2276.0, 2217.0, 1995.0, 2408.0, 2306.0, 2584.0, 2373.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 1983, "step_interval": 5, "values": [951.0, 1294.0, 1060.0, 971.0, 901.0, 1117.0, 1205.0, 1364.0, 1468.0, 1319.0, 1539.0, 1911.0, 2180.0, 1576.0, 2216.0, 1925.0, 2038.0, 2028.0, 2476.0, 2015.0, 2201.0, 2215.0, 2438.0, 3135.0, 2444.0, 2806.0, 2540.0, 2188.0, 2052.0, 2885.0, 2408.0, 3553.0, 2417.0, 2497.0, 2486.0, 3667.0, 2116.0, 2243.0, 2127.0, 2649.0, 3818.0, 2985.0, 2311.0, 2810.0, 2580.0, 2214.0, 2672.0, 2502.0, 2376.0, 2941.0, 3128.0, 2507.0, 2600.0, 2152.0, 2790.0, 3240.0, 2769.0, 2720.0, 2392.0, 3522.0, 2236.0, 2883.0, 2397.0, 2586.0, 2219.0, 3154.0, 2799.0, 2803.0, 2345.0, 2563.0, 2171.0, 2874.0, 2837.0, 2656.0, 3389.0, 2526.0, 2817.0, 2625.0, 3000.0, 2814.0, 2754.0, 2414.0, 3081.0, 2380.0, 2876.0, 2737.0, 2780.0, 2271.0, 2333.0, 2839.0, 2519.0, 3210.0, 2404.0, 2291.0, 2433.0, 2383.0, 2435.0, 1919.0, 2351.0, 2585.0, 2779.0, 2221.0, 2014.0, 2114.0, 1881.0, 2304.0, 2397.0, 2309.0, 2239.0, 2116.0, 2239.0, 2377.0, 2323.0, 2496.0, 2298.0, 2773.0, 2696.0, 1952.0, 2435.0, 2042.0, 2813.0, 2452.0, 2068.0, 2032.0, 2127.0, 2176.0, 2056.0, 2569.0, 2495.0, 2156.0, 2202.0, 2372.0, 2368.0, 2313.0, 1956.0, 2287.0, 2471.0, 2251.0, 2132.0, 1626.0, 2076.0, 2288.0, 2009.0, 1987.0, 2433.0, 1651.0, 2033.0, 2061.0, 1927.0, 2837.0, 2589.0, 2063.0, 1738.0, 1964.0, 2334.0, 1899.0, 2516.0, 2136.0, 2214.0, 1965.0, 1875.0, 2415.0, 1921.0, 2352.0, 2174.0, 1887.0, 2165.0, 2616.0, 1911.0, 1825.0, 1959.0, 1908.0, 1822.0, 1574.0, 1545.0, 2160.0, 1942.0, 2081.0, 1733.0, 2008.0, 2010.0, 2212.0, 1875.0, 1390.0, 1972.0, 2540.0, 1825.0, 2152.0, 1632.0, 2232.0, 1792.0, 1887.0, 1971.0, 2046.0, 1779.0, 2139.0, 2024.0, 1999.0, 1614.0, 1985.0, 1902.0, 2128.0, 2445.0, 2671.0, 2214.0, 2029.0, 2081.0, 2209.0, 2226.0, 1957.0, 2210.0, 2419.0, 2685.0, 2294.0, 1932.0, 2118.0, 1963.0, 1818.0, 1841.0, 2149.0, 2110.0, 2155.0, 1868.0, 2220.0, 2120.0, 2379.0, 1886.0, 2361.0, 1763.0, 2055.0, 1972.0, 2155.0, 1934.0, 2167.0, 1959.0, 1882.0, 1705.0, 1826.0, 1964.0, 2224.0, 1818.0, 1883.0, 1743.0, 2488.0, 2393.0, 2103.0, 2005.0, 2728.0, 2142.0, 2054.0, 1951.0, 1819.0, 2038.0, 2170.0, 2265.0, 1808.0, 2431.0, 1807.0, 2184.0, 2053.0, 1687.0, 1931.0, 2549.0, 2587.0, 1986.0, 2273.0, 2103.0, 2063.0, 2204.0, 2021.0, 2110.0, 2428.0, 2484.0, 2060.0, 2244.0, 2025.0, 1999.0, 1965.0, 1906.0, 2137.0, 2024.0, 2234.0, 1998.0, 2022.0, 1943.0, 2254.0, 2008.0, 1619.0, 1850.0, 2446.0, 2316.0, 1952.0, 2008.0, 2201.0, 2018.0, 2191.0, 1856.0, 2363.0, 2138.0, 2632.0, 1897.0, 2331.0, 1915.0, 2017.0, 2347.0, 2073.0, 2221.0, 2341.0, 1910.0, 1944.0, 2197.0, 2136.0, 2140.0, 2057.0, 2254.0, 1992.0, 2377.0, 1829.0, 2323.0, 2256.0, 2248.0, 2664.0, 2091.0, 2351.0, 2363.0, 2417.0, 1953.0, 2010.0, 2111.0, 2082.0, 2141.0, 2449.0, 2394.0, 2165.0, 2019.0, 2307.0, 2446.0, 2932.0, 2123.0, 2428.0, 2294.0, 2499.0, 2597.0, 2391.0, 2142.0, 2085.0, 2112.0, 2498.0, 2172.0, 2546.0, 2086.0, 2278.0, 2000.0, 2060.0, 2222.0, 2327.0, 2377.0, 2181.0, 1943.0, 2370.0, 2170.0, 2277.0, 2360.0, 2822.0, 2306.0, 2709.0, 2210.0, 2127.0, 2321.0, 2202.0, 2780.0, 2249.0, 2312.0, 2033.0, 2114.0, 2287.0, 2292.0, 2301.0, 2735.0, 2674.0, 2246.0, 2584.0, 2280.0, 2624.0, 2634.0, 2653.0, 2502.0, 2748.0, 2256.0, 2492.0, 2276.0, 2217.0, 1995.0, 2408.0, 2306.0, 2584.0, 2373.0]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.62692]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.62692]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [277.80627]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [277.80627]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/model_config.yaml
index 6cea248b75..e28cc2ba9b 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_bf16_baseline/model_config.yaml
@@ -41,8 +41,8 @@ MODEL_ARGS:
   --pipeline-model-parallel-size: 1
   --deterministic-mode: true
   --no-gradient-accumulation-fusion: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --fp16: true
   --apply-query-key-layer-scaling: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/golden_values.json
new file mode 100644
index 0000000000..6a88c3a850
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [18.14133, 0.62524, 0.62888, 0.61879, 0.62017, 0.62262, 0.61644, 0.95648, 0.62134, 0.62122, 0.62167, 0.61736, 0.62014, 0.61878, 0.61905, 0.62285, 0.62143, 0.62527, 0.6188, 0.61821, 0.62092, 0.6146, 0.62538, 0.62186, 0.62062, 0.61709, 0.61586, 0.62211, 0.62113, 0.62256, 0.91616, 0.62172, 0.62082, 0.61854, 0.61851, 0.61865, 0.61838, 0.62057, 0.62054, 0.62121, 0.62279, 0.61565, 0.61588, 0.61809, 0.6285, 0.62159, 0.619, 0.62096, 0.6161, 0.61341, 0.61939, 0.61863, 0.61901, 0.69973, 0.62205, 0.6203, 0.62205, 0.61913, 0.61593, 0.61268, 0.62209, 0.62242, 0.62178, 0.61463, 0.61723, 0.61562, 0.62222, 0.61147, 0.61537, 0.61793, 0.61712, 0.61962, 0.62226, 0.73426, 0.61519, 0.61809, 0.62057, 0.72077, 0.62008, 0.6196, 0.61771, 0.61875, 0.61628, 0.61618, 0.61608, 0.61962, 0.61838, 0.61834, 0.61866, 0.62047, 0.61852, 0.61278, 0.61478, 0.61796, 0.61939, 0.61855, 0.61816, 0.61585, 0.72525, 0.61589, 0.71497, 0.61452, 0.61899, 0.61647, 0.61769, 0.61448, 0.6133, 0.6161, 0.61341, 0.61318, 0.61661, 0.61966, 0.61316, 0.61487, 0.61573, 0.61347, 0.61386, 0.61593, 0.61745, 0.6185, 0.61792, 0.61356, 0.61533, 0.61644, 0.70276, 0.61398, 0.6159, 0.61832, 0.61774, 0.61711, 0.61411, 0.61533, 0.62272, 0.61709, 0.61557, 0.61705, 0.61893, 0.6177, 0.61888, 0.62207, 0.6181, 0.61501, 0.61758, 0.61994, 0.62402, 0.61667, 0.61599, 0.62131, 0.62011, 0.73481, 0.61752, 0.6206, 0.61654, 0.62124, 0.61775, 0.61832, 0.62597, 0.61901, 0.6153, 0.61393, 0.62147, 0.62628, 0.62091, 0.61689, 0.61436, 0.61683, 0.61743, 0.62116, 0.62033, 0.71198, 0.71973, 0.62179, 0.61968, 0.62104, 0.73504, 0.61833, 0.62098, 0.61898, 0.62766, 0.61917, 0.61475, 0.61706, 0.62025, 0.62046, 0.62146, 0.61796, 0.61756, 0.61818, 0.61889, 0.61869, 0.61959, 0.61761, 0.79997, 0.71316, 0.7092, 0.61693, 0.61553, 0.61793, 0.62191, 0.61846, 0.60521, 0.63066, 0.62491, 0.6225, 0.62102, 0.62456, 0.6247, 0.6269, 0.62537, 0.62411, 0.6231, 0.62397, 0.61873, 0.61766, 0.72647, 0.61878, 0.70741, 0.62227, 0.71605, 0.62022, 0.61781, 0.62597, 0.62427, 0.73275, 0.61764, 0.62069, 0.61913, 0.61957, 0.62075, 0.61693, 0.62163, 0.62496, 0.62065, 0.61855, 0.62534, 0.62563, 0.63027, 0.62765, 0.62046, 0.62782, 0.6225, 0.62116, 0.71019, 0.62081, 0.62867, 0.61875, 0.61378, 0.61727, 0.6238, 0.62162, 0.62088, 0.61962, 0.62082, 0.62352, 0.62164, 0.62001, 0.62139, 0.62, 0.62818, 0.6266, 0.63112, 0.62627, 0.62702, 0.62774, 0.62831, 0.62063, 0.71258, 0.62584, 0.63033, 0.62439, 0.62649, 0.61461, 0.6209, 0.61667, 0.62067, 0.61793, 0.61954, 0.61977, 0.622, 0.6288, 0.62767, 0.62589, 0.62912, 0.62368, 0.61631, 0.73714, 0.6313, 0.61624, 0.61414, 0.62482, 0.6265, 0.62661, 0.62057, 0.62063, 0.62436, 0.62886, 0.62643, 0.62055, 0.61891, 0.62228, 0.62509, 0.62152, 0.62371, 0.62145, 0.61596, 0.62278, 0.62635, 0.63114, 0.72659, 0.72093, 0.62818, 0.62831, 0.61965, 0.62825, 0.62531, 0.6239, 0.6269, 0.6223, 0.62369, 0.62215, 0.62376, 0.62336, 0.62681, 0.62299, 0.62046, 0.61497, 0.61616, 0.61762, 0.62291, 0.61731, 0.61644, 0.61524, 0.61842, 0.62286, 0.61327, 0.61596, 0.6185, 0.61983, 0.62272, 0.61746, 0.6207, 0.6179, 0.61849, 0.62196, 0.62408, 0.62953, 0.62672, 0.62606, 0.61511, 0.61549, 0.6159, 0.62334, 0.62662, 0.75567, 0.62523, 0.62516, 0.62916, 0.62575, 0.62292, 0.62685, 0.62432, 0.62244, 0.61921, 0.61816, 0.61641, 0.61968, 0.62202, 0.6208, 0.6193, 0.61995, 0.62245, 0.61844, 0.61724, 0.61904, 0.61874, 0.62205, 0.6161, 0.61772, 0.70649, 0.62431, 0.61921, 0.62093, 0.61887, 0.62189, 0.62184, 0.62081, 0.62021, 0.62093, 0.62086, 0.62164, 0.6235, 0.61872, 0.62062, 0.61908, 0.62491, 0.62732, 0.62504, 0.61899, 0.62006, 0.6215]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [9.27215, 0.36134, 0.36093, 0.35232, 0.35362, 0.35668, 0.35229, 0.68753, 0.35087, 0.35407, 0.35147, 0.35356, 0.35146, 0.35384, 0.35274, 0.35595, 0.35404, 0.35262, 0.35078, 0.34962, 0.35338, 0.34834, 0.35424, 0.35549, 0.35524, 0.34948, 0.35114, 0.35465, 0.35306, 0.35417, 0.64338, 0.35253, 0.35038, 0.34824, 0.3516, 0.35295, 0.35334, 0.3507, 0.3518, 0.35354, 0.35258, 0.3508, 0.35045, 0.35367, 0.35832, 0.35222, 0.35029, 0.35265, 0.35179, 0.34702, 0.35321, 0.35445, 0.35177, 0.43752, 0.35531, 0.35287, 0.3529, 0.34925, 0.35154, 0.34648, 0.34908, 0.35314, 0.34798, 0.3481, 0.35014, 0.35038, 0.35008, 0.34793, 0.34843, 0.35226, 0.35123, 0.34921, 0.351, 0.46524, 0.34642, 0.35022, 0.34926, 0.45533, 0.35075, 0.35197, 0.34952, 0.35294, 0.35156, 0.35367, 0.35231, 0.35148, 0.34881, 0.34904, 0.35192, 0.35269, 0.35151, 0.34592, 0.34953, 0.35046, 0.35109, 0.35197, 0.35201, 0.34972, 0.45764, 0.34845, 0.44993, 0.34761, 0.35227, 0.34673, 0.35005, 0.34603, 0.34781, 0.34961, 0.34726, 0.3482, 0.3514, 0.35199, 0.34526, 0.3478, 0.35064, 0.34875, 0.35162, 0.34733, 0.3494, 0.34825, 0.35136, 0.34918, 0.34966, 0.34867, 0.43767, 0.34863, 0.35097, 0.35094, 0.34677, 0.35081, 0.35072, 0.35015, 0.35172, 0.35213, 0.34826, 0.34865, 0.35048, 0.3496, 0.34911, 0.35588, 0.35342, 0.35191, 0.35141, 0.35102, 0.35709, 0.34876, 0.34872, 0.35106, 0.35322, 0.46707, 0.35188, 0.35176, 0.35, 0.35379, 0.3509, 0.35081, 0.3551, 0.35093, 0.34933, 0.34848, 0.35167, 0.35398, 0.34723, 0.34792, 0.34845, 0.34775, 0.35079, 0.34957, 0.35345, 0.44501, 0.45138, 0.34891, 0.35082, 0.3502, 0.46589, 0.35255, 0.35187, 0.35127, 0.35483, 0.35059, 0.34896, 0.34861, 0.35247, 0.35179, 0.34935, 0.35234, 0.34933, 0.35334, 0.34686, 0.35171, 0.35547, 0.35168, 0.52709, 0.44719, 0.44161, 0.34936, 0.34954, 0.35313, 0.34988, 0.35211, 0.33688, 0.35591, 0.3569, 0.35308, 0.35372, 0.35241, 0.35314, 0.35633, 0.353, 0.35616, 0.35467, 0.35273, 0.3514, 0.35129, 0.45541, 0.3499, 0.44221, 0.35081, 0.44665, 0.35109, 0.35024, 0.35427, 0.35423, 0.46289, 0.34881, 0.35173, 0.34964, 0.35399, 0.35206, 0.35147, 0.35326, 0.35451, 0.35111, 0.35112, 0.35937, 0.35913, 0.36067, 0.35939, 0.35289, 0.35237, 0.34936, 0.35284, 0.44138, 0.35073, 0.35858, 0.35425, 0.34953, 0.35087, 0.35453, 0.35091, 0.35251, 0.34904, 0.35282, 0.35193, 0.35492, 0.35161, 0.35115, 0.35118, 0.36151, 0.35849, 0.36407, 0.35821, 0.36041, 0.35561, 0.36252, 0.35429, 0.44699, 0.36096, 0.36201, 0.35407, 0.35747, 0.35035, 0.35103, 0.34874, 0.35637, 0.3524, 0.35102, 0.35202, 0.35462, 0.35968, 0.35397, 0.35259, 0.35547, 0.35321, 0.35018, 0.46643, 0.3583, 0.35092, 0.34697, 0.3538, 0.35589, 0.35223, 0.35164, 0.35261, 0.35967, 0.36013, 0.35806, 0.35023, 0.35024, 0.3526, 0.34984, 0.35259, 0.35298, 0.35284, 0.35138, 0.35036, 0.35288, 0.35847, 0.45332, 0.44559, 0.35561, 0.35336, 0.3521, 0.35312, 0.35227, 0.35234, 0.35359, 0.35468, 0.35224, 0.35204, 0.35651, 0.35583, 0.35358, 0.35435, 0.35427, 0.3497, 0.35079, 0.35172, 0.35517, 0.35178, 0.35126, 0.34889, 0.35033, 0.35332, 0.34892, 0.35261, 0.35094, 0.35215, 0.35764, 0.35341, 0.35384, 0.35265, 0.35263, 0.35262, 0.35604, 0.36288, 0.35642, 0.35552, 0.3484, 0.34851, 0.3514, 0.36023, 0.35789, 0.48902, 0.36035, 0.36141, 0.3626, 0.35908, 0.35622, 0.35631, 0.35269, 0.35075, 0.35039, 0.35096, 0.35039, 0.34953, 0.35289, 0.34822, 0.35154, 0.35088, 0.35383, 0.35072, 0.34872, 0.34826, 0.34902, 0.35267, 0.34801, 0.34971, 0.43955, 0.35085, 0.34994, 0.35373, 0.34855, 0.3492, 0.35231, 0.34725, 0.35003, 0.3473, 0.35104, 0.34755, 0.34992, 0.35186, 0.35388, 0.35074, 0.34993, 0.35194, 0.35167, 0.34626, 0.35392, 0.35198]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.80897, 0.28475, 0.2809, 0.27885, 0.27971, 0.2768, 0.27791, 0.2813, 0.2828, 0.27982, 0.28277, 0.27676, 0.28261, 0.27806, 0.28033, 0.2756, 0.28082, 0.27955, 0.28018, 0.27766, 0.27802, 0.27721, 0.28203, 0.27953, 0.27943, 0.27922, 0.27814, 0.28056, 0.28107, 0.27624, 0.28037, 0.28169, 0.2828, 0.28312, 0.28074, 0.27837, 0.27679, 0.28303, 0.2829, 0.28043, 0.27823, 0.27266, 0.27336, 0.27459, 0.28023, 0.27652, 0.27746, 0.2779, 0.27563, 0.27401, 0.27717, 0.27499, 0.27806, 0.27139, 0.27365, 0.27659, 0.28082, 0.28038, 0.27531, 0.27517, 0.28057, 0.27667, 0.28628, 0.27883, 0.27588, 0.27536, 0.27984, 0.2729, 0.27334, 0.27425, 0.27422, 0.27613, 0.27623, 0.2746, 0.27458, 0.27341, 0.27807, 0.27236, 0.27663, 0.27538, 0.27514, 0.27306, 0.2725, 0.27083, 0.27026, 0.27509, 0.27586, 0.27515, 0.27392, 0.27389, 0.27372, 0.2727, 0.27096, 0.27354, 0.27409, 0.27274, 0.27274, 0.27361, 0.27352, 0.27457, 0.27411, 0.27589, 0.27459, 0.27704, 0.27375, 0.27488, 0.27373, 0.27473, 0.27336, 0.27408, 0.27412, 0.27621, 0.27573, 0.2757, 0.27319, 0.27286, 0.27081, 0.27628, 0.27632, 0.27773, 0.27459, 0.27302, 0.27391, 0.27706, 0.27302, 0.27235, 0.2728, 0.27422, 0.27771, 0.27408, 0.273, 0.27313, 0.27881, 0.2727, 0.27535, 0.27554, 0.27602, 0.27445, 0.27748, 0.27334, 0.27196, 0.27246, 0.27334, 0.2765, 0.27324, 0.27646, 0.27446, 0.27758, 0.27638, 0.2749, 0.27379, 0.27822, 0.27586, 0.27434, 0.27452, 0.2751, 0.27681, 0.27448, 0.27334, 0.27477, 0.27831, 0.27967, 0.28117, 0.27795, 0.27331, 0.27527, 0.27361, 0.27892, 0.27512, 0.27366, 0.27646, 0.27988, 0.27713, 0.27762, 0.27574, 0.27463, 0.27934, 0.27654, 0.28122, 0.27818, 0.27487, 0.27565, 0.27548, 0.27639, 0.27869, 0.27377, 0.27686, 0.2737, 0.27871, 0.27425, 0.27333, 0.27386, 0.27879, 0.2752, 0.27707, 0.27628, 0.27433, 0.27416, 0.28211, 0.27328, 0.27772, 0.2888, 0.28238, 0.28559, 0.28328, 0.28926, 0.29069, 0.28744, 0.28541, 0.28383, 0.28569, 0.28878, 0.28294, 0.28177, 0.28457, 0.28391, 0.27915, 0.28556, 0.28795, 0.28723, 0.28157, 0.28876, 0.288, 0.28233, 0.28245, 0.28563, 0.28586, 0.27943, 0.28324, 0.27971, 0.28335, 0.28509, 0.28373, 0.28221, 0.27996, 0.2821, 0.28282, 0.28146, 0.2827, 0.29287, 0.28819, 0.28375, 0.28224, 0.28618, 0.28593, 0.27803, 0.2775, 0.27939, 0.28305, 0.28516, 0.28387, 0.28394, 0.27989, 0.28606, 0.28244, 0.28311, 0.2822, 0.28452, 0.28083, 0.28371, 0.27966, 0.28404, 0.27905, 0.28671, 0.28017, 0.28042, 0.27826, 0.27799, 0.28104, 0.28485, 0.2833, 0.27803, 0.28505, 0.28078, 0.27731, 0.27811, 0.2825, 0.2845, 0.28366, 0.28285, 0.29128, 0.28986, 0.28737, 0.28519, 0.28008, 0.28508, 0.29026, 0.27934, 0.27842, 0.28735, 0.28334, 0.29041, 0.28444, 0.28192, 0.27975, 0.28248, 0.28157, 0.28471, 0.28418, 0.28337, 0.29038, 0.28525, 0.28937, 0.28336, 0.28092, 0.28765, 0.2938, 0.28931, 0.28955, 0.29117, 0.29147, 0.29048, 0.28242, 0.29224, 0.28996, 0.28762, 0.28995, 0.28361, 0.28955, 0.28314, 0.28125, 0.28279, 0.28923, 0.28566, 0.28096, 0.27889, 0.27987, 0.28102, 0.28378, 0.27825, 0.27822, 0.28139, 0.28151, 0.284, 0.28038, 0.27763, 0.28234, 0.28237, 0.27877, 0.27839, 0.28213, 0.27969, 0.27977, 0.28461, 0.28193, 0.28295, 0.28539, 0.28439, 0.28043, 0.28021, 0.27978, 0.27678, 0.28057, 0.28152, 0.27875, 0.27736, 0.28042, 0.28071, 0.27701, 0.28009, 0.28081, 0.28054, 0.27846, 0.27695, 0.27435, 0.28018, 0.27863, 0.2831, 0.27711, 0.27774, 0.27798, 0.27776, 0.27805, 0.27924, 0.27943, 0.27863, 0.27639, 0.27628, 0.27471, 0.28218, 0.2775, 0.27692, 0.28008, 0.28228, 0.27856, 0.28233, 0.27871, 0.28388, 0.27878, 0.2831, 0.28268, 0.27716, 0.2756, 0.27712, 0.28343, 0.28463, 0.28241, 0.28327, 0.27551, 0.27892]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.62041, 0.00418, 0.00386, 0.00419, 0.00438, 0.0044, 0.00464, 0.00467, 0.00468, 0.00448, 0.00443, 0.00436, 0.00461, 0.00452, 0.00471, 0.00475, 0.00426, 0.00443, 0.00451, 0.00448, 0.00454, 0.00422, 0.00444, 0.00458, 0.00446, 0.00447, 0.00432, 0.00458, 0.00459, 0.00455, 0.00456, 0.0044, 0.00451, 0.00445, 0.00465, 0.00435, 0.00439, 0.00431, 0.00431, 0.00453, 0.0045, 0.00449, 0.00456, 0.00437, 0.00432, 0.0043, 0.00442, 0.0045, 0.0042, 0.00427, 0.0045, 0.00438, 0.00447, 0.00452, 0.0046, 0.00429, 0.00439, 0.00441, 0.00462, 0.00448, 0.00409, 0.00434, 0.00448, 0.0042, 0.00454, 0.00422, 0.00431, 0.00413, 0.00439, 0.00414, 0.00456, 0.00464, 0.00426, 0.00434, 0.00414, 0.00453, 0.00423, 0.00453, 0.00431, 0.00403, 0.00414, 0.0043, 0.00446, 0.00423, 0.00437, 0.00434, 0.00419, 0.0042, 0.00433, 0.00435, 0.00443, 0.00408, 0.00416, 0.00451, 0.00443, 0.00435, 0.00446, 0.00421, 0.00467, 0.00454, 0.00431, 0.00462, 0.00433, 0.00426, 0.00437, 0.00437, 0.00433, 0.00435, 0.00426, 0.00413, 0.00435, 0.00422, 0.00431, 0.00432, 0.0043, 0.00408, 0.00435, 0.00438, 0.00439, 0.00426, 0.00438, 0.00432, 0.00449, 0.00423, 0.00444, 0.00436, 0.00417, 0.00424, 0.0042, 0.00428, 0.00425, 0.00425, 0.0042, 0.00445, 0.0043, 0.00429, 0.00441, 0.0043, 0.00412, 0.00429, 0.0042, 0.00419, 0.0042, 0.00427, 0.00427, 0.00418, 0.00464, 0.00406, 0.00435, 0.0046, 0.0043, 0.00438, 0.00417, 0.00427, 0.0044, 0.00444, 0.0045, 0.00407, 0.00421, 0.00403, 0.00442, 0.00418, 0.00425, 0.00425, 0.00434, 0.00422, 0.00432, 0.00446, 0.00435, 0.00452, 0.00428, 0.00408, 0.00445, 0.00414, 0.00441, 0.00412, 0.00434, 0.00445, 0.00425, 0.00412, 0.00432, 0.00441, 0.00432, 0.00422, 0.00429, 0.00407, 0.00434, 0.00448, 0.00434, 0.00434, 0.00423, 0.00422, 0.0046, 0.00418, 0.00445, 0.00432, 0.00422, 0.00418, 0.00408, 0.00434, 0.03441, 0.00493, 0.00506, 0.00555, 0.00518, 0.00512, 0.00537, 0.00513, 0.00501, 0.00506, 0.00504, 0.00473, 0.00488, 0.00523, 0.00528, 0.00511, 0.00526, 0.00496, 0.00546, 0.00512, 0.0054, 0.00539, 0.00514, 0.00484, 0.00515, 0.00531, 0.00515, 0.00498, 0.00509, 0.0051, 0.00516, 0.00496, 0.00494, 0.00501, 0.00511, 0.00536, 0.00517, 0.00549, 0.00531, 0.00526, 0.00531, 0.00497, 0.00498, 0.00524, 0.00486, 0.00502, 0.00497, 0.00491, 0.00509, 0.00466, 0.00519, 0.00528, 0.00486, 0.00509, 0.0049, 0.005, 0.00508, 0.005, 0.00503, 0.00473, 0.00536, 0.00516, 0.00549, 0.00528, 0.00506, 0.00513, 0.00501, 0.00563, 0.00498, 0.00498, 0.0051, 0.00528, 0.00509, 0.005, 0.00495, 0.00509, 0.00508, 0.00485, 0.00479, 0.00485, 0.00507, 0.00499, 0.00463, 0.00497, 0.00487, 0.00529, 0.00518, 0.00483, 0.00513, 0.0051, 0.005, 0.005, 0.00514, 0.00496, 0.00492, 0.00547, 0.00506, 0.00502, 0.00481, 0.0051, 0.00498, 0.0051, 0.00475, 0.00498, 0.0048, 0.00528, 0.00523, 0.0053, 0.00561, 0.00522, 0.00517, 0.00528, 0.00505, 0.00511, 0.00538, 0.00531, 0.00528, 0.00554, 0.00534, 0.00512, 0.00541, 0.00533, 0.00508, 0.00518, 0.00519, 0.00548, 0.00545, 0.00554, 0.0052, 0.00506, 0.00513, 0.00502, 0.00523, 0.00513, 0.00478, 0.00487, 0.00503, 0.00512, 0.0051, 0.00529, 0.005, 0.00521, 0.00528, 0.00511, 0.00522, 0.00513, 0.00533, 0.00502, 0.0053, 0.00492, 0.00522, 0.00496, 0.00488, 0.00513, 0.00506, 0.00519, 0.00508, 0.00521, 0.00442, 0.00409, 0.00426, 0.0043, 0.00418, 0.00428, 0.00456, 0.00443, 0.00422, 0.00426, 0.0043, 0.00429, 0.00435, 0.00446, 0.0044, 0.00447, 0.00444, 0.0043, 0.0042, 0.00438, 0.00422, 0.00429, 0.00463, 0.00435, 0.00431, 0.00447, 0.00431, 0.00441, 0.00417, 0.00425, 0.0044, 0.00438, 0.00438, 0.00439, 0.00447, 0.00402, 0.00423, 0.00447, 0.00451, 0.00457, 0.00458, 0.00426]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.22336, 0.00298, 0.00292, 0.00297, 0.0029, 0.00289, 0.00306, 0.00314, 0.00321, 0.003, 0.00296, 0.00297, 0.00294, 0.00288, 0.00301, 0.00324, 0.00323, 0.00298, 0.00292, 0.00298, 0.00295, 0.0029, 0.00308, 0.00319, 0.00324, 0.00299, 0.00292, 0.00301, 0.00293, 0.00291, 0.00326, 0.00322, 0.00323, 0.0029, 0.00293, 0.003, 0.00291, 0.00287, 0.00303, 0.0032, 0.00322, 0.00298, 0.00294, 0.00295, 0.00296, 0.0029, 0.00305, 0.00322, 0.00321, 0.003, 0.00295, 0.00299, 0.00295, 0.00292, 0.00306, 0.00323, 0.0032, 0.00298, 0.00291, 0.00297, 0.00296, 0.00287, 0.00304, 0.00322, 0.0032, 0.00299, 0.00296, 0.00297, 0.00296, 0.00291, 0.00308, 0.00321, 0.00326, 0.00301, 0.00294, 0.00292, 0.00295, 0.00287, 0.00307, 0.00321, 0.00318, 0.00296, 0.00285, 0.00302, 0.00297, 0.00291, 0.003, 0.00323, 0.0032, 0.003, 0.00292, 0.00294, 0.00297, 0.00285, 0.00306, 0.00318, 0.00314, 0.003, 0.00289, 0.00296, 0.00296, 0.00288, 0.00307, 0.00321, 0.00321, 0.00301, 0.00289, 0.00297, 0.00297, 0.0029, 0.00298, 0.00323, 0.00321, 0.003, 0.00289, 0.00287, 0.00295, 0.00292, 0.00302, 0.00323, 0.00323, 0.003, 0.00292, 0.00291, 0.00298, 0.00286, 0.00306, 0.00321, 0.00322, 0.00302, 0.00289, 0.00293, 0.00286, 0.00288, 0.00306, 0.00322, 0.00319, 0.00295, 0.00285, 0.00297, 0.00295, 0.00289, 0.00305, 0.0032, 0.00324, 0.00298, 0.00291, 0.00297, 0.00289, 0.00289, 0.00304, 0.0032, 0.00314, 0.003, 0.00289, 0.00297, 0.00295, 0.00288, 0.00301, 0.00317, 0.00314, 0.003, 0.00291, 0.00299, 0.00296, 0.0029, 0.00306, 0.00324, 0.00319, 0.00301, 0.0029, 0.00296, 0.00296, 0.0029, 0.00306, 0.00319, 0.0032, 0.003, 0.00285, 0.00298, 0.00296, 0.00281, 0.00305, 0.00318, 0.00322, 0.00297, 0.00291, 0.00299, 0.00294, 0.00292, 0.00307, 0.00323, 0.00324, 0.00299, 0.0029, 0.00299, 0.00295, 0.0029, 0.00305, 0.00319, 0.0029, 0.00305, 0.00311, 0.00325, 0.00324, 0.00308, 0.00284, 0.00305, 0.00295, 0.00305, 0.003, 0.00324, 0.0032, 0.00306, 0.00286, 0.00306, 0.00294, 0.00305, 0.0031, 0.00318, 0.00323, 0.00308, 0.00288, 0.00306, 0.00297, 0.00304, 0.00309, 0.00321, 0.00322, 0.00308, 0.00287, 0.00299, 0.00294, 0.00304, 0.00311, 0.00324, 0.00325, 0.00304, 0.00281, 0.00302, 0.00293, 0.00307, 0.0031, 0.00323, 0.00319, 0.00306, 0.00286, 0.00306, 0.00291, 0.00305, 0.00311, 0.00314, 0.00323, 0.00303, 0.00285, 0.00298, 0.00294, 0.00302, 0.00307, 0.00322, 0.00318, 0.00303, 0.00287, 0.00303, 0.00294, 0.00301, 0.00322, 0.00321, 0.00326, 0.00304, 0.00288, 0.00305, 0.00292, 0.00304, 0.00303, 0.00323, 0.00323, 0.00307, 0.00289, 0.003, 0.00295, 0.00298, 0.00307, 0.00328, 0.00312, 0.00307, 0.00289, 0.00303, 0.00294, 0.00306, 0.00309, 0.00324, 0.0032, 0.00306, 0.0029, 0.00306, 0.00294, 0.00301, 0.00301, 0.00322, 0.00321, 0.00306, 0.00289, 0.00304, 0.00293, 0.00303, 0.00312, 0.00322, 0.00325, 0.00305, 0.00286, 0.00306, 0.00293, 0.00304, 0.0031, 0.00325, 0.00326, 0.00306, 0.00287, 0.00305, 0.00296, 0.00307, 0.00314, 0.00315, 0.00323, 0.00307, 0.00288, 0.00293, 0.0029, 0.00303, 0.00304, 0.00325, 0.00322, 0.00304, 0.0028, 0.00304, 0.00292, 0.00305, 0.00308, 0.00323, 0.00323, 0.00307, 0.00289, 0.00304, 0.00294, 0.00305, 0.00311, 0.00321, 0.00322, 0.00303, 0.00281, 0.00304, 0.00296, 0.003, 0.0031, 0.00322, 0.00314, 0.00301, 0.00281, 0.00298, 0.00288, 0.00303, 0.00307, 0.00321, 0.0032, 0.00301, 0.00281, 0.00303, 0.00288, 0.00301, 0.00309, 0.00316, 0.00319, 0.00302, 0.00284, 0.00306, 0.00292, 0.003, 0.00328, 0.00321, 0.0032, 0.00301, 0.00285, 0.00297, 0.00284, 0.003, 0.003, 0.00318, 0.00319, 0.00301, 0.00281, 0.00303, 0.00289, 0.003, 0.00305, 0.00315, 0.00308, 0.00303, 0.00279, 0.00299]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0004, 0.00019, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00026, 0.00027, 0.00028, 0.00028, 0.00029, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00031, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00029, 0.00029, 0.00029, 0.00029, 0.00027, 0.00029, 0.00027, 0.00028, 0.00028, 0.00028, 0.00029, 0.00027, 0.00027, 0.00029, 0.00028, 0.0003, 0.00028, 0.00028, 0.00028, 0.00028, 0.00029, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00028, 0.00028, 0.00028, 0.00027, 0.00028, 0.00027, 0.00028, 0.00026, 0.00026, 0.00026, 0.00026, 0.00026, 0.00026, 0.00027, 0.00027, 0.00025, 0.00025, 0.00027, 0.00028, 0.00027, 0.00028, 0.00026, 0.00026, 0.00025, 0.00026, 0.00026, 0.00028, 0.00025, 0.00028, 0.00027, 0.00026, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00026, 0.00028, 0.00029, 0.00028, 0.00028, 0.00028, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00027, 0.00028, 0.00027, 0.00028, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00029, 0.00027, 0.00028, 0.00028, 0.00028, 0.00027, 0.00027, 0.00027, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00027, 0.00027, 0.00028, 0.00027, 0.00027, 0.00027, 0.00028, 0.00029, 0.00028, 0.00028, 0.00027, 0.00028, 0.00027, 0.00028, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00028, 0.00027, 0.00028, 0.00028, 0.00029, 0.00027, 0.00028, 0.00027, 0.00027, 0.00029, 0.00028, 0.00028, 0.00027, 0.00028, 0.00028, 0.00027, 0.00028, 0.00029, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00029, 0.00028, 0.00028, 0.00029, 0.00028, 0.00028, 0.00029, 0.00028, 0.00028, 0.00028, 0.00028, 0.00027, 0.00026, 0.00026, 0.00026, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00027, 0.00025, 0.00025, 0.00026, 0.00026, 0.00025, 0.00027, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00027, 0.00025, 0.00025, 0.00025, 0.00027, 0.00027, 0.00025, 0.00025, 0.00025, 0.00026, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00027, 0.00027, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00027, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00026, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00026, 0.00027, 0.00029, 0.00027, 0.00027, 0.00028, 0.00027, 0.00028, 0.00028, 0.00029, 0.00028, 0.00027, 0.00027, 0.00027, 0.00027, 0.00028, 0.00027, 0.00027, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00025, 0.00027, 0.00025, 0.00027, 0.00028, 0.00027, 0.00027, 0.00027, 0.00027, 0.00028, 0.00027, 0.00028, 0.00027, 0.00027, 0.00027, 0.00027]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.6202, 0.00104, 0.00121, 0.00115, 0.00122, 0.00121, 0.00123, 0.00124, 0.00122, 0.00123, 0.00125, 0.00122, 0.00121, 0.0012, 0.00122, 0.00127, 0.00121, 0.00123, 0.0012, 0.00123, 0.00121, 0.00116, 0.00125, 0.00122, 0.00122, 0.00124, 0.00122, 0.00123, 0.0012, 0.00122, 0.00125, 0.00122, 0.00126, 0.0012, 0.00122, 0.00123, 0.00121, 0.00127, 0.00121, 0.00121, 0.00121, 0.00121, 0.00123, 0.00122, 0.00123, 0.00124, 0.00121, 0.0012, 0.00122, 0.00119, 0.00121, 0.00122, 0.00137, 0.00122, 0.00121, 0.00123, 0.0012, 0.00126, 0.00121, 0.00122, 0.00122, 0.00129, 0.00122, 0.00122, 0.00122, 0.00123, 0.00125, 0.00125, 0.00124, 0.00122, 0.00123, 0.0013, 0.00124, 0.00121, 0.00123, 0.00118, 0.00123, 0.00121, 0.00123, 0.00118, 0.00118, 0.00118, 0.00119, 0.00119, 0.00119, 0.00121, 0.00121, 0.00122, 0.00121, 0.00123, 0.00123, 0.0012, 0.00128, 0.00117, 0.00122, 0.00123, 0.00124, 0.00121, 0.00118, 0.00119, 0.00121, 0.00122, 0.00121, 0.0012, 0.00118, 0.00124, 0.00122, 0.0012, 0.00125, 0.0012, 0.00121, 0.00101, 0.0012, 0.00121, 0.00124, 0.00123, 0.00123, 0.00123, 0.00122, 0.001, 0.00122, 0.00121, 0.001, 0.00125, 0.00122, 0.00121, 0.00124, 0.00121, 0.00121, 0.00099, 0.0012, 0.00125, 0.00121, 0.001, 0.0012, 0.00122, 0.00122, 0.00122, 0.0013, 0.00097, 0.00124, 0.00122, 0.00125, 0.00121, 0.0012, 0.0012, 0.00121, 0.00123, 0.0012, 0.0012, 0.00121, 0.00125, 0.00135, 0.00122, 0.00122, 0.00123, 0.00124, 0.00121, 0.00122, 0.0012, 0.0013, 0.00122, 0.00124, 0.001, 0.00123, 0.00121, 0.00121, 0.00126, 0.00124, 0.00129, 0.00129, 0.00124, 0.00121, 0.00119, 0.0012, 0.00123, 0.00123, 0.00127, 0.00122, 0.00122, 0.0012, 0.00121, 0.00128, 0.0012, 0.00125, 0.00124, 0.00121, 0.00123, 0.00121, 0.00132, 0.00122, 0.00121, 0.0012, 0.00122, 0.00123, 0.00123, 0.00121, 0.0012, 0.00122, 0.00123, 0.0012, 0.00123, 0.0012, 0.00118, 0.00118, 0.00121, 0.00124, 0.0012, 0.00121, 0.00121, 0.00119, 0.00119, 0.0012, 0.0012, 0.0012, 0.00118, 0.00126, 0.00121, 0.00118, 0.0012, 0.00117, 0.00119, 0.00121, 0.00118, 0.00119, 0.00122, 0.0012, 0.0012, 0.00126, 0.00121, 0.00128, 0.00107, 0.00115, 0.00121, 0.00119, 0.00119, 0.00116, 0.00118, 0.0012, 0.00121, 0.00119, 0.0012, 0.0012, 0.0012, 0.00116, 0.00121, 0.0012, 0.00116, 0.00121, 0.00113, 0.00119, 0.00127, 0.0012, 0.00119, 0.00118, 0.00119, 0.0012, 0.00121, 0.00119, 0.00118, 0.00119, 0.0012, 0.00119, 0.0012, 0.0012, 0.00127, 0.00122, 0.0012, 0.00118, 0.00118, 0.00121, 0.00118, 0.00123, 0.00119, 0.00122, 0.00116, 0.0012, 0.00118, 0.0012, 0.00122, 0.00122, 0.00121, 0.00117, 0.00121, 0.00117, 0.0012, 0.00118, 0.00119, 0.00122, 0.00118, 0.00125, 0.00119, 0.00121, 0.00118, 0.00133, 0.00119, 0.00119, 0.00119, 0.0012, 0.00128, 0.00121, 0.00122, 0.0012, 0.00123, 0.00115, 0.00118, 0.0012, 0.00122, 0.00119, 0.00122, 0.00121, 0.00119, 0.00126, 0.0012, 0.0012, 0.00118, 0.00116, 0.00119, 0.00118, 0.00121, 0.00119, 0.00125, 0.00122, 0.00119, 0.00116, 0.00117, 0.00119, 0.0012, 0.0012, 0.00117, 0.00118, 0.0012, 0.00124, 0.00122, 0.0012, 0.00118, 0.0012, 0.00119, 0.0012, 0.00118, 0.00119, 0.00121, 0.00119, 0.00119, 0.00121, 0.00118, 0.00126, 0.00118, 0.0012, 0.00119, 0.00117, 0.0012, 0.00118, 0.0012, 0.00119, 0.0012, 0.00119, 0.00125, 0.00117, 0.00123, 0.00118, 0.00122, 0.00122, 0.00122, 0.00117, 0.00123, 0.00122, 0.00121, 0.00121, 0.0012, 0.00121, 0.00128, 0.00123, 0.00116, 0.0012, 0.00123, 0.00123, 0.00116, 0.00123, 0.00121, 0.0012, 0.00121, 0.00122, 0.00124, 0.00128, 0.00122, 0.00117, 0.00123, 0.00124, 0.00122, 0.00118, 0.0012, 0.00117, 0.00125, 0.00122, 0.00117, 0.00115, 0.00118, 0.00113, 0.0012]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00555, 0.00512, 0.0052, 0.0051, 0.00517, 0.00513, 0.00514, 0.00513, 0.00512, 0.00511, 0.00508, 0.0051, 0.0051, 0.00512, 0.00511, 0.00509, 0.00508, 0.00511, 0.00514, 0.0051, 0.00509, 0.0051, 0.00514, 0.00512, 0.00512, 0.00512, 0.00514, 0.00517, 0.00511, 0.00513, 0.00513, 0.00516, 0.00515, 0.00515, 0.00516, 0.00514, 0.00513, 0.00543, 0.00514, 0.00512, 0.00514, 0.00513, 0.00513, 0.00516, 0.00512, 0.00515, 0.00511, 0.00513, 0.00515, 0.00514, 0.0051, 0.00512, 0.0057, 0.00511, 0.00513, 0.00513, 0.00514, 0.0053, 0.00514, 0.00511, 0.00513, 0.00512, 0.00513, 0.00518, 0.00513, 0.00514, 0.00512, 0.00513, 0.00512, 0.00509, 0.00512, 0.00539, 0.00514, 0.00514, 0.0051, 0.00512, 0.00511, 0.00512, 0.00511, 0.00511, 0.00512, 0.00513, 0.00511, 0.00514, 0.00512, 0.0051, 0.00514, 0.00511, 0.00512, 0.00522, 0.0051, 0.00514, 0.00572, 0.0051, 0.00515, 0.00526, 0.00509, 0.00511, 0.00513, 0.00513, 0.00518, 0.00514, 0.00511, 0.00512, 0.00512, 0.00511, 0.00514, 0.00512, 0.00518, 0.00514, 0.00512, 0.00513, 0.00512, 0.00512, 0.00512, 0.00511, 0.00509, 0.00514, 0.00519, 0.00512, 0.0051, 0.00513, 0.0051, 0.00548, 0.00514, 0.00512, 0.00512, 0.00511, 0.00511, 0.00512, 0.00511, 0.00519, 0.00533, 0.00509, 0.00512, 0.0051, 0.00513, 0.00511, 0.00515, 0.00508, 0.00512, 0.00513, 0.0057, 0.00513, 0.00513, 0.00516, 0.00518, 0.00515, 0.00517, 0.00513, 0.00514, 0.00516, 0.0057, 0.00516, 0.00515, 0.00514, 0.00513, 0.00513, 0.00516, 0.00516, 0.00566, 0.00514, 0.00514, 0.00515, 0.00516, 0.00515, 0.00513, 0.00517, 0.00513, 0.00513, 0.00601, 0.00514, 0.00522, 0.00513, 0.00515, 0.00514, 0.00517, 0.00511, 0.00515, 0.00516, 0.00515, 0.00514, 0.00515, 0.00512, 0.00587, 0.00517, 0.00518, 0.00516, 0.00513, 0.00541, 0.00514, 0.00515, 0.00513, 0.00516, 0.00521, 0.00531, 0.00532, 0.00517, 0.00516, 0.00515, 0.00511, 0.00529, 0.00509, 0.00511, 0.00512, 0.00512, 0.00512, 0.00515, 0.0053, 0.0051, 0.00512, 0.00512, 0.00512, 0.00511, 0.0051, 0.00513, 0.00512, 0.00513, 0.00513, 0.00512, 0.00559, 0.00511, 0.0051, 0.0051, 0.00512, 0.00515, 0.00512, 0.00511, 0.00579, 0.00512, 0.00511, 0.00512, 0.00511, 0.00511, 0.00511, 0.00513, 0.00508, 0.00513, 0.00511, 0.00509, 0.00512, 0.0051, 0.00512, 0.00511, 0.00512, 0.00513, 0.00511, 0.00514, 0.00511, 0.00512, 0.00512, 0.0059, 0.00513, 0.00514, 0.00512, 0.00511, 0.00513, 0.00511, 0.00511, 0.0051, 0.00509, 0.0051, 0.00512, 0.0051, 0.0051, 0.00511, 0.00513, 0.00513, 0.0051, 0.00513, 0.00511, 0.0051, 0.0051, 0.00511, 0.00512, 0.00511, 0.00509, 0.00513, 0.0051, 0.0051, 0.00518, 0.0051, 0.00513, 0.00509, 0.00513, 0.00512, 0.00511, 0.00515, 0.00512, 0.00512, 0.00512, 0.00512, 0.00512, 0.00511, 0.00601, 0.00512, 0.00524, 0.00512, 0.0051, 0.00511, 0.00509, 0.00512, 0.0051, 0.00512, 0.00511, 0.00511, 0.00526, 0.0051, 0.00511, 0.00512, 0.00511, 0.00511, 0.00514, 0.00511, 0.00512, 0.00509, 0.00511, 0.00512, 0.00512, 0.00509, 0.0051, 0.00511, 0.00511, 0.00513, 0.00512, 0.00541, 0.00512, 0.00515, 0.00511, 0.00509, 0.0051, 0.00512, 0.00511, 0.00512, 0.00511, 0.00517, 0.00514, 0.00513, 0.00513, 0.00512, 0.00511, 0.00514, 0.00511, 0.00514, 0.00509, 0.00508, 0.00513, 0.00509, 0.0051, 0.00513, 0.00511, 0.00571, 0.00519, 0.00511, 0.00511, 0.0051, 0.00511, 0.00512, 0.00513, 0.00511, 0.00511, 0.00511, 0.00511, 0.00512, 0.00511, 0.00509, 0.00514, 0.00511, 0.00516, 0.00512, 0.0053, 0.00511, 0.00512, 0.00521, 0.00512, 0.00513, 0.00514, 0.00512, 0.00512, 0.00514, 0.0051, 0.00511, 0.00513, 0.00512, 0.00509, 0.00519, 0.00512, 0.0051, 0.00509, 0.00596, 0.00512, 0.0051, 0.0051, 0.00513, 0.00513, 0.0051, 0.00511, 0.00509, 0.00512, 0.00511]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00417, 0.00096, 0.00098, 0.00098, 0.00099, 0.00097, 0.00098, 0.00098, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00099, 0.00098, 0.00097, 0.00098, 0.00097, 0.00097, 0.00096, 0.00098, 0.00098, 0.00099, 0.00099, 0.00097, 0.00096, 0.00098, 0.00098, 0.00101, 0.00097, 0.00097, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00098, 0.00098, 0.00098, 0.00096, 0.00097, 0.00097, 0.00097, 0.00098, 0.00098, 0.00097, 0.00097, 0.00097, 0.00098, 0.00098, 0.00099, 0.00098, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00098, 0.00097, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00098, 0.00096, 0.00097, 0.00097, 0.00098, 0.00096, 0.00096, 0.00097, 0.00098, 0.00096, 0.00097, 0.00096, 0.00097, 0.00099, 0.00096, 0.00098, 0.00098, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00099, 0.00096, 0.00096, 0.00097, 0.00097, 0.00097, 0.00098, 0.00096, 0.00097, 0.00097, 0.00098, 0.00099, 0.00098, 0.00096, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00098, 0.00099, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00098, 0.00097, 0.00098, 0.00097, 0.00096, 0.00097, 0.00099, 0.00098, 0.00097, 0.00098, 0.00096, 0.00097, 0.00097, 0.00096, 0.00097, 0.00098, 0.00099, 0.00097, 0.00097, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00099, 0.00098, 0.00097, 0.00097, 0.00098, 0.00097, 0.00096, 0.00097, 0.00098, 0.00097, 0.001, 0.00097, 0.00097, 0.00097, 0.00098, 0.00097, 0.00098, 0.00097, 0.00099, 0.00097, 0.00097, 0.00096, 0.00098, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00098, 0.00098, 0.00097, 0.00097, 0.00099, 0.00097, 0.00098, 0.00098, 0.00097, 0.00097, 0.00098, 0.00098, 0.001, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00096, 0.00096, 0.00097, 0.001, 0.00096, 0.00099, 0.00097, 0.00098, 0.00097, 0.00099, 0.00096, 0.00128, 0.00096, 0.00096, 0.00097, 0.00097, 0.00097, 0.00099, 0.00096, 0.00097, 0.00096, 0.00097, 0.00097, 0.00096, 0.00096, 0.00096, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00098, 0.00097, 0.00097, 0.00096, 0.00097, 0.001, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00096, 0.00097, 0.00096, 0.00096, 0.00096, 0.00097, 0.001, 0.00097, 0.00096, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00096, 0.00096, 0.00096, 0.00097, 0.00099, 0.00096, 0.00097, 0.00096, 0.00096, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00098, 0.00097, 0.00097, 0.00099, 0.00096, 0.00097, 0.00096, 0.00096, 0.00098, 0.00096, 0.00096, 0.00097, 0.00098, 0.00096, 0.00097, 0.00097, 0.00096, 0.00098, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00097, 0.00096, 0.00097, 0.00097, 0.00096, 0.00096, 0.00097, 0.00096, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00096, 0.00096, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00096, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00097, 0.00095, 0.00096, 0.00097, 0.00098, 0.00097, 0.00097, 0.00097, 0.00097, 0.00096, 0.00096, 0.00096, 0.00098, 0.00097, 0.00097, 0.00098, 0.00097, 0.00098, 0.00098, 0.00098, 0.00098, 0.001, 0.00098, 0.00098, 0.00098, 0.00097, 0.00097, 0.00098, 0.00098, 0.00101, 0.00098, 0.00098, 0.00097, 0.00098, 0.00097, 0.00097, 0.00099, 0.00097, 0.00098, 0.00098, 0.00096, 0.00098, 0.00097, 0.00098, 0.00099, 0.00097, 0.00098, 0.00097, 0.00097, 0.00098, 0.00098]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00118, 0.00099, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.001, 0.001, 0.00101, 0.00101, 0.00101, 0.00103, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00102, 0.00101, 0.001, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.001, 0.00102, 0.00102, 0.001, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00101, 0.001, 0.001, 0.00101, 0.00102, 0.00102, 0.001, 0.00101, 0.001, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.00105, 0.00101, 0.00102, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.001, 0.001, 0.00102, 0.001, 0.00101, 0.00102, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.00103, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00101, 0.00106, 0.00101, 0.00102, 0.00101, 0.00101, 0.00101, 0.001, 0.00101, 0.001, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00102, 0.00102, 0.00102, 0.00101, 0.00101, 0.00102, 0.001, 0.00106, 0.00101, 0.00101, 0.00101, 0.00101, 0.00101, 0.00103, 0.00102, 0.00101, 0.00102, 0.00101, 0.00101, 0.00101, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.00102, 0.00101, 0.001, 0.001, 0.001, 0.001, 0.001, 0.00101, 0.00101, 0.00101, 0.00102, 0.00102, 0.00101, 0.00102, 0.00103, 0.00102, 0.00101, 0.00101, 0.00102, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00103, 0.00101, 0.00101, 0.00101, 0.00101, 0.00102, 0.00102, 0.00101, 0.00102, 0.00101, 0.00101, 0.00102, 0.00102, 0.00102, 0.00105, 0.00102, 0.00102, 0.00101, 0.00101, 0.00102, 0.00101, 0.00103, 0.00102, 0.00102, 0.00101, 0.00106, 0.00102, 0.00101, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00108, 0.00102, 0.00104, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00107, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00107, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00104, 0.00102, 0.00104, 0.00102, 0.00102, 0.00103, 0.00103, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00101, 0.00103, 0.00101, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00105, 0.00102, 0.00102, 0.00104, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00101, 0.00103, 0.00104, 0.00103, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00108, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00122, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00101, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00101, 0.00102, 0.00103, 0.00103, 0.00103, 0.00101, 0.00102, 0.00102, 0.00102, 0.00101, 0.00101, 0.00105, 0.00102, 0.00103, 0.00102, 0.00102, 0.00101, 0.00102, 0.00102, 0.00102, 0.00101, 0.00101, 0.00101, 0.00102, 0.00101, 0.00101, 0.00102, 0.00102, 0.00102, 0.00101, 0.00102, 0.00103, 0.00101, 0.00102, 0.00102, 0.00102, 0.00102, 0.00101, 0.00104, 0.00102, 0.00102, 0.00102, 0.00102, 0.00101, 0.00102, 0.00102, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00102, 0.00103, 0.00102, 0.00102, 0.00101, 0.00102, 0.00101, 0.00101]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.63386, 0.00867, 0.00903, 0.00886, 0.00906, 0.00897, 0.00901, 0.009, 0.00896, 0.00895, 0.00895, 0.00895, 0.00894, 0.00894, 0.00896, 0.009, 0.00892, 0.00896, 0.00899, 0.00897, 0.00892, 0.00887, 0.00902, 0.00897, 0.009, 0.00906, 0.00899, 0.00902, 0.00897, 0.00898, 0.0091, 0.00901, 0.00904, 0.00898, 0.00901, 0.009, 0.00902, 0.00937, 0.00899, 0.00896, 0.00901, 0.00897, 0.00899, 0.00902, 0.00897, 0.00903, 0.00895, 0.00898, 0.00899, 0.00895, 0.00896, 0.00898, 0.00978, 0.00897, 0.00898, 0.009, 0.00895, 0.0092, 0.00896, 0.00901, 0.009, 0.00904, 0.00898, 0.00902, 0.00897, 0.00899, 0.00902, 0.00902, 0.00899, 0.00899, 0.00898, 0.00934, 0.00904, 0.00896, 0.00897, 0.00891, 0.00895, 0.00892, 0.00894, 0.0089, 0.00889, 0.0089, 0.00891, 0.00892, 0.00888, 0.0089, 0.009, 0.00896, 0.00895, 0.0091, 0.00889, 0.00892, 0.00967, 0.00886, 0.009, 0.00913, 0.00896, 0.00896, 0.00889, 0.00895, 0.00901, 0.00899, 0.00903, 0.00893, 0.00893, 0.00898, 0.009, 0.00894, 0.00905, 0.00897, 0.00894, 0.00877, 0.00897, 0.00898, 0.00902, 0.00895, 0.00895, 0.009, 0.00905, 0.00875, 0.00895, 0.00897, 0.00872, 0.00942, 0.00901, 0.00898, 0.00897, 0.00894, 0.00895, 0.00876, 0.00895, 0.00907, 0.00917, 0.00872, 0.00895, 0.00893, 0.00898, 0.00897, 0.00906, 0.00866, 0.00896, 0.00897, 0.00964, 0.00897, 0.00897, 0.00898, 0.009, 0.009, 0.009, 0.00894, 0.00898, 0.00904, 0.00977, 0.00905, 0.00899, 0.00901, 0.00905, 0.00898, 0.00901, 0.00898, 0.00965, 0.009, 0.009, 0.00878, 0.00905, 0.00899, 0.00898, 0.00904, 0.00902, 0.00906, 0.01008, 0.00901, 0.00907, 0.00895, 0.00899, 0.00902, 0.00905, 0.00902, 0.00902, 0.00901, 0.00899, 0.00898, 0.00908, 0.00899, 0.00979, 0.00905, 0.00904, 0.00903, 0.009, 0.00938, 0.00899, 0.00901, 0.00904, 0.00902, 0.00909, 0.00923, 0.00917, 0.00901, 0.00905, 0.00903, 0.00899, 0.00918, 0.00889, 0.00891, 0.00894, 0.00894, 0.00896, 0.00895, 0.00912, 0.00892, 0.00889, 0.00896, 0.0089, 0.00891, 0.00901, 0.0089, 0.00904, 0.00893, 0.00893, 0.00894, 0.00942, 0.00889, 0.00938, 0.00887, 0.00892, 0.00897, 0.00893, 0.00896, 0.00974, 0.00891, 0.009, 0.00879, 0.00886, 0.00891, 0.0089, 0.00892, 0.00885, 0.00891, 0.0089, 0.00892, 0.00896, 0.0089, 0.00892, 0.00893, 0.00891, 0.00894, 0.00892, 0.00891, 0.00894, 0.00885, 0.00891, 0.00986, 0.00894, 0.00893, 0.00892, 0.00894, 0.00896, 0.00889, 0.00893, 0.00888, 0.0089, 0.00891, 0.0089, 0.0089, 0.00894, 0.00901, 0.00902, 0.00898, 0.00887, 0.00892, 0.00897, 0.00888, 0.00894, 0.00889, 0.00893, 0.00887, 0.00889, 0.00895, 0.00891, 0.00891, 0.00904, 0.00901, 0.00889, 0.00892, 0.00891, 0.00892, 0.00891, 0.00892, 0.00895, 0.00891, 0.00902, 0.00891, 0.00892, 0.00889, 0.01004, 0.00891, 0.00907, 0.00893, 0.00889, 0.00901, 0.00889, 0.00893, 0.00895, 0.00898, 0.00885, 0.00891, 0.00914, 0.00891, 0.00891, 0.00894, 0.00892, 0.00888, 0.009, 0.0089, 0.00948, 0.00889, 0.00887, 0.00893, 0.00889, 0.00889, 0.00891, 0.00896, 0.00894, 0.00893, 0.00888, 0.00921, 0.00895, 0.00893, 0.00894, 0.00887, 0.0089, 0.00897, 0.00896, 0.00894, 0.00893, 0.00896, 0.009, 0.00892, 0.00897, 0.00891, 0.00889, 0.00895, 0.0089, 0.00893, 0.00891, 0.00886, 0.009, 0.00888, 0.00889, 0.00894, 0.00885, 0.00955, 0.00901, 0.00895, 0.00891, 0.0089, 0.00889, 0.00898, 0.00888, 0.00898, 0.00889, 0.00895, 0.00895, 0.00896, 0.00891, 0.00895, 0.00904, 0.00897, 0.00901, 0.00897, 0.00919, 0.00904, 0.00899, 0.00902, 0.00895, 0.00901, 0.00901, 0.00892, 0.00909, 0.00899, 0.00896, 0.00901, 0.00899, 0.009, 0.00896, 0.00905, 0.0089, 0.00897, 0.00898, 0.00984, 0.00894, 0.00894, 0.00891, 0.00903, 0.00898, 0.00894, 0.00889, 0.0089, 0.0089, 0.00894]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.88321, 10.90268, 10.88687, 10.83314, 10.67636, 10.64925, 10.43407, 10.15143, 9.939, 9.84142, 9.58871, 9.85432, 9.88466, 9.62953, 9.78812, 9.5115, 9.45845, 9.64924, 9.38622, 9.33216, 9.24226, 9.14549, 9.17557, 8.99547, 9.18942, 9.05996, 9.15554, 9.16495, 9.29785, 8.98464, 8.92921, 9.04391, 9.04317, 8.65502, 8.71709, 8.75344, 8.68371, 8.7343, 8.65869, 8.76488, 8.66084, 8.84969, 8.83212, 8.4992, 8.38905, 8.43151, 8.49327, 8.38449, 8.43266, 8.57974, 8.36712, 8.19218, 8.22599, 8.22213, 8.26761, 7.91363, 8.09574, 7.89107, 8.2463, 8.23044, 8.00478, 7.9653, 7.91788, 7.73983, 7.73952, 7.64266, 7.51535, 7.9067, 7.6981, 7.45174, 7.74028, 7.76751, 7.54113, 7.29838, 7.45192, 7.33549, 7.46187, 7.22351, 7.63653, 7.27884, 7.35151, 7.2129, 7.2187, 7.42237, 7.17713, 7.28373, 7.00153, 7.00528, 7.04066, 7.1397, 6.8246, 6.98624, 7.08901, 7.00075, 6.87398, 6.75446, 6.98902, 7.05484, 6.70056, 6.57618, 6.7239, 6.73842, 6.73087, 6.73636, 6.65702, 6.40579, 6.6386, 6.62005, 6.44721, 6.63067, 6.74344, 6.6111, 6.7266, 6.69523, 6.62503, 6.50683, 6.59892, 6.4067, 6.66402, 6.24864, 6.25205, 6.30302, 6.38991, 6.35064, 6.45057, 6.2892, 6.34021, 6.23934, 6.20441, 6.39672, 6.32669, 6.3228, 6.16602, 6.15875, 6.24058, 6.38585, 6.20055, 6.14534, 6.17669, 6.1094, 6.05525, 6.06665, 6.2527, 6.40409, 6.25252, 6.2934, 6.0919, 6.17395, 5.99575, 6.02272, 5.94996, 6.23797, 6.18154, 5.95877, 5.77498, 6.11727, 5.84271, 6.09751, 5.78563, 6.15394, 6.14296, 6.08411, 5.92729, 6.11238, 5.94309, 6.19339, 5.89494, 5.792, 5.77614, 5.6837, 6.01618, 5.99613, 6.06338, 5.88778, 6.04018, 5.96996, 5.99544, 5.98695, 5.94778, 5.84144, 5.95287, 5.61942, 5.70133, 5.88893, 5.84402, 5.86128, 5.76114, 5.83707, 5.72343, 5.55889, 5.72351, 5.62534, 5.83303, 5.60569, 5.7102, 5.70991, 5.89681, 5.64325, 5.84924, 5.73928, 5.87114, 5.33228, 5.89693, 5.872, 5.85316, 5.40988, 5.4088, 5.62665, 5.59641, 5.48639, 5.57896, 5.67332, 5.47579, 5.74541, 5.50851, 5.59461, 5.621, 5.62129, 5.51073, 5.61357, 5.67793, 5.68632, 5.58943, 5.66035, 5.37294, 5.67985, 5.62736, 5.42133, 5.58734, 5.63109, 5.55307, 5.34119, 5.53841, 5.48634, 5.48174, 5.37484, 5.55776, 5.60342, 5.38738, 5.52728, 5.4859, 5.33181, 5.50554, 5.40833, 5.44, 5.31717, 5.06482, 5.47629, 5.56511, 5.71212, 5.41184, 5.59499, 5.63272, 5.23153, 5.27192, 5.3912, 5.39311, 5.32484, 5.49539, 5.18175, 5.29693, 5.24506, 5.37468, 5.25384, 5.44332, 5.53548, 5.3125, 5.43753, 5.3339, 5.07, 5.31161, 5.25178, 5.30057, 5.1086, 5.27262, 5.26395, 5.46902, 5.15667, 5.26704, 5.20746, 5.35466, 4.98016, 4.91076, 5.3213, 5.39019, 5.22162, 5.3164, 5.10162, 5.1553, 5.25943, 5.06435, 5.26075, 5.07101, 5.33638, 5.24297, 5.14623, 5.23826, 5.03699, 5.31101, 5.04764, 5.02142, 5.13778, 5.10838, 5.26722, 5.14671, 5.27266, 5.09162, 5.0919, 5.24829, 5.3185, 5.25029, 5.18579, 5.14206, 5.28335, 4.94328, 5.20523, 5.08657, 5.29719, 5.17312, 5.18231, 5.10943, 4.98051, 4.99195, 5.21896, 5.30825, 5.09051, 5.05174, 4.91264, 5.11732, 5.11518, 4.92322, 5.33386, 5.02007, 5.09792, 5.16007, 4.99811, 5.05898, 5.06488, 4.98971, 5.07389, 5.15699, 4.97292, 5.17835, 4.92646, 4.91925, 5.06679, 4.99198, 4.90773, 4.77047, 4.93905, 5.10914, 5.0148, 5.01342, 5.32728, 4.95518, 4.99041, 5.04238, 4.79783, 4.72965, 4.99227, 5.0394, 4.87169, 4.95051, 5.03887, 5.01995, 4.81482, 4.88854, 4.89947, 4.82779, 4.74234, 5.00778, 4.7467, 5.20619, 4.78181, 4.98955, 4.73414, 4.78105, 4.81703, 4.64628, 4.65374, 4.83873, 4.80327, 4.79812, 4.9214, 4.87849, 4.92132, 4.76615, 4.87858, 4.72843, 4.9077, 4.95342, 4.86965, 4.70236, 4.77862, 4.89666, 4.70572, 4.85677, 4.68692, 4.68192, 4.64505]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.88321, 10.90268, 10.88687, 10.83314, 10.67636, 10.64925, 10.43407, 10.15143, 9.939, 9.84142, 9.58871, 9.85432, 9.88466, 9.62953, 9.78812, 9.5115, 9.45845, 9.64924, 9.38622, 9.33216, 9.24226, 9.14549, 9.17557, 8.99547, 9.18942, 9.05996, 9.15554, 9.16495, 9.29785, 8.98464, 8.92921, 9.04391, 9.04317, 8.65502, 8.71709, 8.75344, 8.68371, 8.7343, 8.65869, 8.76488, 8.66084, 8.84969, 8.83212, 8.4992, 8.38905, 8.43151, 8.49327, 8.38449, 8.43266, 8.57974, 8.36712, 8.19218, 8.22599, 8.22213, 8.26761, 7.91363, 8.09574, 7.89107, 8.2463, 8.23044, 8.00478, 7.9653, 7.91788, 7.73983, 7.73952, 7.64266, 7.51535, 7.9067, 7.6981, 7.45174, 7.74028, 7.76751, 7.54113, 7.29838, 7.45192, 7.33549, 7.46187, 7.22351, 7.63653, 7.27884, 7.35151, 7.2129, 7.2187, 7.42237, 7.17713, 7.28373, 7.00153, 7.00528, 7.04066, 7.1397, 6.8246, 6.98624, 7.08901, 7.00075, 6.87398, 6.75446, 6.98902, 7.05484, 6.70056, 6.57618, 6.7239, 6.73842, 6.73087, 6.73636, 6.65702, 6.40579, 6.6386, 6.62005, 6.44721, 6.63067, 6.74344, 6.6111, 6.7266, 6.69523, 6.62503, 6.50683, 6.59892, 6.4067, 6.66402, 6.24864, 6.25205, 6.30302, 6.38991, 6.35064, 6.45057, 6.2892, 6.34021, 6.23934, 6.20441, 6.39672, 6.32669, 6.3228, 6.16602, 6.15875, 6.24058, 6.38585, 6.20055, 6.14534, 6.17669, 6.1094, 6.05525, 6.06665, 6.2527, 6.40409, 6.25252, 6.2934, 6.0919, 6.17395, 5.99575, 6.02272, 5.94996, 6.23797, 6.18154, 5.95877, 5.77498, 6.11727, 5.84271, 6.09751, 5.78563, 6.15394, 6.14296, 6.08411, 5.92729, 6.11238, 5.94309, 6.19339, 5.89494, 5.792, 5.77614, 5.6837, 6.01618, 5.99613, 6.06338, 5.88778, 6.04018, 5.96996, 5.99544, 5.98695, 5.94778, 5.84144, 5.95287, 5.61942, 5.70133, 5.88893, 5.84402, 5.86128, 5.76114, 5.83707, 5.72343, 5.55889, 5.72351, 5.62534, 5.83303, 5.60569, 5.7102, 5.70991, 5.89681, 5.64325, 5.84924, 5.73928, 5.87114, 5.33228, 5.89693, 5.872, 5.85316, 5.40988, 5.4088, 5.62665, 5.59641, 5.48639, 5.57896, 5.67332, 5.47579, 5.74541, 5.50851, 5.59461, 5.621, 5.62129, 5.51073, 5.61357, 5.67793, 5.68632, 5.58943, 5.66035, 5.37294, 5.67985, 5.62736, 5.42133, 5.58734, 5.63109, 5.55307, 5.34119, 5.53841, 5.48634, 5.48174, 5.37484, 5.55776, 5.60342, 5.38738, 5.52728, 5.4859, 5.33181, 5.50554, 5.40833, 5.44, 5.31717, 5.06482, 5.47629, 5.56511, 5.71212, 5.41184, 5.59499, 5.63272, 5.23153, 5.27192, 5.3912, 5.39311, 5.32484, 5.49539, 5.18175, 5.29693, 5.24506, 5.37468, 5.25384, 5.44332, 5.53548, 5.3125, 5.43753, 5.3339, 5.07, 5.31161, 5.25178, 5.30057, 5.1086, 5.27262, 5.26395, 5.46902, 5.15667, 5.26704, 5.20746, 5.35466, 4.98016, 4.91076, 5.3213, 5.39019, 5.22162, 5.3164, 5.10162, 5.1553, 5.25943, 5.06435, 5.26075, 5.07101, 5.33638, 5.24297, 5.14623, 5.23826, 5.03699, 5.31101, 5.04764, 5.02142, 5.13778, 5.10838, 5.26722, 5.14671, 5.27266, 5.09162, 5.0919, 5.24829, 5.3185, 5.25029, 5.18579, 5.14206, 5.28335, 4.94328, 5.20523, 5.08657, 5.29719, 5.17312, 5.18231, 5.10943, 4.98051, 4.99195, 5.21896, 5.30825, 5.09051, 5.05174, 4.91264, 5.11732, 5.11518, 4.92322, 5.33386, 5.02007, 5.09792, 5.16007, 4.99811, 5.05898, 5.06488, 4.98971, 5.07389, 5.15699, 4.97292, 5.17835, 4.92646, 4.91925, 5.06679, 4.99198, 4.90773, 4.77047, 4.93905, 5.10914, 5.0148, 5.01342, 5.32728, 4.95518, 4.99041, 5.04238, 4.79783, 4.72965, 4.99227, 5.0394, 4.87169, 4.95051, 5.03887, 5.01995, 4.81482, 4.88854, 4.89947, 4.82779, 4.74234, 5.00778, 4.7467, 5.20619, 4.78181, 4.98955, 4.73414, 4.78105, 4.81703, 4.64628, 4.65374, 4.83873, 4.80327, 4.79812, 4.9214, 4.87849, 4.92132, 4.76615, 4.87858, 4.72843, 4.9077, 4.95342, 4.86965, 4.70236, 4.77862, 4.89666, 4.70572, 4.85677, 4.68692, 4.68192, 4.64505]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [12.95641, 13.2384, 13.63492, 12.46753, 12.09519, 9.48185, 7.05331, 7.26898, 6.13791, 4.65533, 4.16677, 2.85409, 2.39258, 2.35693, 2.05902, 2.22136, 2.15373, 1.91319, 2.28507, 2.08136, 2.12587, 2.16293, 2.01255, 2.22443, 1.98488, 2.10576, 1.90696, 1.9543, 1.94666, 2.19132, 2.07534, 1.9973, 1.90676, 2.17071, 2.13949, 2.12242, 2.00142, 1.85779, 1.93941, 1.74128, 2.19131, 1.80266, 1.76804, 1.92184, 1.89627, 1.81829, 1.73892, 1.73316, 1.7548, 1.56741, 1.70661, 1.78909, 1.75371, 1.8099, 1.69083, 1.80378, 1.72805, 1.87537, 1.64718, 1.47793, 1.64751, 1.54177, 1.73678, 1.93709, 1.70003, 1.61404, 1.65733, 1.60718, 1.41019, 1.66006, 1.44415, 1.3449, 1.59801, 1.38078, 1.40657, 1.58642, 1.37384, 1.47591, 1.51235, 1.32276, 1.27695, 1.35665, 1.39793, 1.46181, 1.25641, 1.39278, 1.37555, 1.31206, 1.25327, 1.08729, 1.11608, 1.26073, 1.05493, 1.26676, 1.03825, 1.22449, 1.31527, 1.17458, 1.05643, 1.32651, 1.60257, 1.2771, 1.33646, 1.31918, 1.248, 1.20478, 1.17877, 1.39792, 1.21711, 1.31304, 1.06851, 0.90225, 1.00231, 1.02701, 1.08335, 1.06592, 1.11157, 1.35469, 1.11475, 0.96782, 1.00793, 1.10818, 0.98621, 1.2088, 1.33881, 1.44029, 1.6209, 1.4596, 1.76932, 0.95989, 1.18019, 1.10796, 1.01963, 0.97229, 1.12326, 1.18955, 1.04787, 1.17124, 1.15064, 0.95989, 1.2251, 1.2379, 1.76155, 1.26203, 1.48837, 1.2467, 1.12532, 1.2807, 1.00776, 1.29835, 1.39203, 1.19636, 1.4484, 1.31191, 1.0452, 1.72246, 1.72833, 1.28959, 1.84591, 1.35158, 1.59884, 1.36455, 1.22883, 0.94147, 1.4872, 1.47058, 1.60177, 1.17187, 1.32032, 1.16147, 1.85664, 1.34438, 1.41884, 1.939, 1.3293, 1.75251, 1.4942, 1.19914, 1.25112, 1.47923, 1.19903, 1.70249, 1.28382, 1.22996, 1.38428, 1.04416, 1.49206, 1.45812, 1.5496, 1.42558, 1.5666, 1.60373, 1.50198, 2.14466, 1.64657, 1.23816, 1.19399, 1.20748, 1.27992, 1.28244, 1.01251, 1.42205, 1.36197, 1.11149, 1.15089, 1.21404, 1.39311, 1.5652, 1.38265, 1.4134, 1.55375, 1.48078, 1.28046, 1.56958, 1.42513, 1.45697, 1.27067, 1.6129, 1.30064, 1.30128, 1.59962, 2.07562, 1.66274, 1.53273, 1.30633, 1.38281, 1.30251, 1.26134, 1.59835, 1.39505, 1.20665, 1.50419, 1.33709, 1.53729, 1.35211, 1.18328, 1.72786, 1.56925, 1.48159, 1.79747, 1.32018, 1.29802, 1.45777, 1.41144, 1.32018, 1.82833, 1.47341, 1.38161, 1.37728, 1.47317, 1.22182, 1.50379, 1.40184, 1.43299, 1.38574, 1.54027, 1.3871, 1.51693, 1.73604, 1.27623, 1.30004, 1.43266, 1.26605, 1.31063, 1.40554, 1.47355, 1.43481, 1.66877, 1.27269, 1.36414, 1.39902, 1.36787, 1.30634, 1.35432, 1.33569, 1.38439, 1.38254, 1.48327, 1.3313, 1.47336, 1.54266, 1.45093, 1.39023, 1.42073, 1.71873, 1.24142, 1.27025, 1.75206, 1.19488, 1.72063, 1.35861, 1.46103, 1.32756, 1.38252, 1.44831, 1.49026, 1.5017, 1.67806, 1.49633, 1.40813, 1.2821, 1.34708, 1.20139, 1.33134, 1.30935, 1.28049, 1.39953, 1.36021, 1.30784, 1.55113, 1.45126, 1.35267, 1.8948, 1.31989, 1.26079, 1.54872, 1.25987, 1.49108, 1.31905, 1.39623, 1.42575, 1.70894, 1.69908, 1.44957, 1.53553, 1.41451, 1.68745, 1.45251, 1.2816, 1.33701, 1.40832, 1.76682, 1.43394, 1.35911, 1.42618, 1.36908, 1.37004, 1.25362, 1.44167, 1.3631, 1.32537, 1.0708, 1.21959, 1.38245, 1.69458, 1.66343, 1.49487, 1.64475, 1.18445, 1.24234, 1.37689, 1.3449, 1.29452, 1.57163, 1.48364, 1.39813, 1.46563, 1.16757, 1.33935, 1.37732, 1.74665, 1.43255, 1.6591, 1.35981, 1.18773, 1.72037, 1.57868, 1.47314, 1.60009, 1.70452, 1.52569, 1.35993, 1.71308, 1.55029, 1.45496, 1.45713, 1.21934, 1.34612, 1.35689, 1.29738, 1.27919, 1.35703, 1.34356, 1.23723, 1.16682, 1.55154, 1.54928, 1.31127, 1.22661, 1.39907, 1.23896, 1.39069, 1.35517, 1.4518, 1.74352, 1.41812, 1.48035, 1.43537, 1.2798, 1.31958]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [12.95641, 13.2384, 13.63492, 12.46753, 12.09519, 9.48185, 7.05331, 7.26898, 6.13791, 4.65533, 4.16677, 2.85409, 2.39258, 2.35693, 2.05902, 2.22136, 2.15373, 1.91319, 2.28507, 2.08136, 2.12587, 2.16293, 2.01255, 2.22443, 1.98488, 2.10576, 1.90696, 1.9543, 1.94666, 2.19132, 2.07534, 1.9973, 1.90676, 2.17071, 2.13949, 2.12242, 2.00142, 1.85779, 1.93941, 1.74128, 2.19131, 1.80266, 1.76804, 1.92184, 1.89627, 1.81829, 1.73892, 1.73316, 1.7548, 1.56741, 1.70661, 1.78909, 1.75371, 1.8099, 1.69083, 1.80378, 1.72805, 1.87537, 1.64718, 1.47793, 1.64751, 1.54177, 1.73678, 1.93709, 1.70003, 1.61404, 1.65733, 1.60718, 1.41019, 1.66006, 1.44415, 1.3449, 1.59801, 1.38078, 1.40657, 1.58642, 1.37384, 1.47591, 1.51235, 1.32276, 1.27695, 1.35665, 1.39793, 1.46181, 1.25641, 1.39278, 1.37555, 1.31206, 1.25327, 1.08729, 1.11608, 1.26073, 1.05493, 1.26676, 1.03825, 1.22449, 1.31527, 1.17458, 1.05643, 1.32651, 1.60257, 1.2771, 1.33646, 1.31918, 1.248, 1.20478, 1.17877, 1.39792, 1.21711, 1.31304, 1.06851, 0.90225, 1.00231, 1.02701, 1.08335, 1.06592, 1.11157, 1.35469, 1.11475, 0.96782, 1.00793, 1.10818, 0.98621, 1.2088, 1.33881, 1.44029, 1.6209, 1.4596, 1.76932, 0.95989, 1.18019, 1.10796, 1.01963, 0.97229, 1.12326, 1.18955, 1.04787, 1.17124, 1.15064, 0.95989, 1.2251, 1.2379, 1.76155, 1.26203, 1.48837, 1.2467, 1.12532, 1.2807, 1.00776, 1.29835, 1.39203, 1.19636, 1.4484, 1.31191, 1.0452, 1.72246, 1.72833, 1.28959, 1.84591, 1.35158, 1.59884, 1.36455, 1.22883, 0.94147, 1.4872, 1.47058, 1.60177, 1.17187, 1.32032, 1.16147, 1.85664, 1.34438, 1.41884, 1.939, 1.3293, 1.75251, 1.4942, 1.19914, 1.25112, 1.47923, 1.19903, 1.70249, 1.28382, 1.22996, 1.38428, 1.04416, 1.49206, 1.45812, 1.5496, 1.42558, 1.5666, 1.60373, 1.50198, 2.14466, 1.64657, 1.23816, 1.19399, 1.20748, 1.27992, 1.28244, 1.01251, 1.42205, 1.36197, 1.11149, 1.15089, 1.21404, 1.39311, 1.5652, 1.38265, 1.4134, 1.55375, 1.48078, 1.28046, 1.56958, 1.42513, 1.45697, 1.27067, 1.6129, 1.30064, 1.30128, 1.59962, 2.07562, 1.66274, 1.53273, 1.30633, 1.38281, 1.30251, 1.26134, 1.59835, 1.39505, 1.20665, 1.50419, 1.33709, 1.53729, 1.35211, 1.18328, 1.72786, 1.56925, 1.48159, 1.79747, 1.32018, 1.29802, 1.45777, 1.41144, 1.32018, 1.82833, 1.47341, 1.38161, 1.37728, 1.47317, 1.22182, 1.50379, 1.40184, 1.43299, 1.38574, 1.54027, 1.3871, 1.51693, 1.73604, 1.27623, 1.30004, 1.43266, 1.26605, 1.31063, 1.40554, 1.47355, 1.43481, 1.66877, 1.27269, 1.36414, 1.39902, 1.36787, 1.30634, 1.35432, 1.33569, 1.38439, 1.38254, 1.48327, 1.3313, 1.47336, 1.54266, 1.45093, 1.39023, 1.42073, 1.71873, 1.24142, 1.27025, 1.75206, 1.19488, 1.72063, 1.35861, 1.46103, 1.32756, 1.38252, 1.44831, 1.49026, 1.5017, 1.67806, 1.49633, 1.40813, 1.2821, 1.34708, 1.20139, 1.33134, 1.30935, 1.28049, 1.39953, 1.36021, 1.30784, 1.55113, 1.45126, 1.35267, 1.8948, 1.31989, 1.26079, 1.54872, 1.25987, 1.49108, 1.31905, 1.39623, 1.42575, 1.70894, 1.69908, 1.44957, 1.53553, 1.41451, 1.68745, 1.45251, 1.2816, 1.33701, 1.40832, 1.76682, 1.43394, 1.35911, 1.42618, 1.36908, 1.37004, 1.25362, 1.44167, 1.3631, 1.32537, 1.0708, 1.21959, 1.38245, 1.69458, 1.66343, 1.49487, 1.64475, 1.18445, 1.24234, 1.37689, 1.3449, 1.29452, 1.57163, 1.48364, 1.39813, 1.46563, 1.16757, 1.33935, 1.37732, 1.74665, 1.43255, 1.6591, 1.35981, 1.18773, 1.72037, 1.57868, 1.47314, 1.60009, 1.70452, 1.52569, 1.35993, 1.71308, 1.55029, 1.45496, 1.45713, 1.21934, 1.34612, 1.35689, 1.29738, 1.27919, 1.35703, 1.34356, 1.23723, 1.16682, 1.55154, 1.54928, 1.31127, 1.22661, 1.39907, 1.23896, 1.39069, 1.35517, 1.4518, 1.74352, 1.41812, 1.48035, 1.43537, 1.2798, 1.31958]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [58.0, 81.0, 78.0, 82.0, 76.0, 95.0, 104.0, 114.0, 114.0, 147.0, 119.0, 159.0, 165.0, 173.0, 182.0, 167.0, 188.0, 176.0, 167.0, 165.0, 187.0, 162.0, 191.0, 164.0, 181.0, 170.0, 168.0, 172.0, 182.0, 180.0, 164.0, 171.0, 169.0, 154.0, 144.0, 172.0, 173.0, 198.0, 168.0, 210.0, 178.0, 156.0, 174.0, 177.0, 163.0, 172.0, 206.0, 172.0, 184.0, 197.0, 223.0, 153.0, 162.0, 187.0, 173.0, 201.0, 146.0, 152.0, 240.0, 231.0, 192.0, 208.0, 162.0, 210.0, 192.0, 282.0, 232.0, 174.0, 215.0, 186.0, 227.0, 258.0, 202.0, 265.0, 192.0, 216.0, 239.0, 200.0, 265.0, 210.0, 264.0, 231.0, 179.0, 221.0, 234.0, 184.0, 188.0, 206.0, 157.0, 228.0, 217.0, 227.0, 219.0, 233.0, 191.0, 187.0, 214.0, 190.0, 237.0, 168.0, 155.0, 174.0, 165.0, 157.0, 155.0, 136.0, 154.0, 133.0, 124.0, 167.0, 187.0, 158.0, 188.0, 161.0, 168.0, 130.0, 164.0, 109.0, 181.0, 166.0, 146.0, 145.0, 130.0, 132.0, 130.0, 145.0, 125.0, 107.0, 130.0, 147.0, 128.0, 137.0, 149.0, 151.0, 133.0, 117.0, 167.0, 153.0, 134.0, 131.0, 117.0, 116.0, 100.0, 125.0, 121.0, 139.0, 125.0, 139.0, 124.0, 118.0, 103.0, 142.0, 95.0, 127.0, 109.0, 102.0, 110.0, 119.0, 101.0, 129.0, 122.0, 143.0, 119.0, 131.0, 102.0, 117.0, 98.0, 140.0, 129.0, 106.0, 76.0, 115.0, 81.0, 87.0, 118.0, 84.0, 101.0, 118.0, 99.0, 99.0, 107.0, 108.0, 137.0, 131.0, 109.0, 123.0, 107.0, 104.0, 102.0, 138.0, 125.0, 119.0, 91.0, 79.0, 87.0, 112.0, 104.0, 98.0, 101.0, 109.0, 135.0, 98.0, 89.0, 117.0, 106.0, 127.0, 103.0, 111.0, 122.0, 102.0, 92.0, 99.0, 110.0, 93.0, 123.0, 114.0, 133.0, 87.0, 114.0, 121.0, 111.0, 95.0, 93.0, 102.0, 127.0, 88.0, 127.0, 114.0, 107.0, 110.0, 101.0, 110.0, 108.0, 99.0, 106.0, 126.0, 92.0, 96.0, 94.0, 77.0, 124.0, 119.0, 91.0, 105.0, 110.0, 103.0, 97.0, 116.0, 104.0, 97.0, 117.0, 92.0, 110.0, 114.0, 97.0, 101.0, 92.0, 105.0, 93.0, 141.0, 93.0, 106.0, 116.0, 107.0, 122.0, 107.0, 128.0, 100.0, 94.0, 105.0, 124.0, 114.0, 94.0, 80.0, 98.0, 105.0, 97.0, 99.0, 132.0, 94.0, 99.0, 93.0, 108.0, 108.0, 107.0, 111.0, 134.0, 114.0, 104.0, 102.0, 123.0, 108.0, 109.0, 107.0, 110.0, 121.0, 92.0, 94.0, 130.0, 128.0, 130.0, 83.0, 110.0, 130.0, 105.0, 99.0, 106.0, 107.0, 101.0, 100.0, 98.0, 131.0, 101.0, 116.0, 89.0, 106.0, 114.0, 115.0, 112.0, 110.0, 128.0, 92.0, 88.0, 112.0, 108.0, 106.0, 83.0, 113.0, 129.0, 126.0, 99.0, 118.0, 98.0, 101.0, 102.0, 103.0, 119.0, 126.0, 128.0, 110.0, 107.0, 128.0, 125.0, 119.0, 113.0, 89.0, 102.0, 103.0, 126.0, 141.0, 95.0, 106.0, 117.0, 109.0, 93.0, 109.0, 111.0, 138.0, 124.0, 114.0, 106.0, 92.0, 109.0, 105.0, 144.0, 122.0, 108.0, 112.0, 86.0, 100.0, 127.0, 108.0, 100.0, 113.0, 99.0, 103.0, 104.0, 96.0, 125.0, 122.0, 97.0, 128.0, 117.0, 121.0, 133.0, 115.0, 95.0, 126.0, 117.0, 136.0, 118.0, 108.0, 135.0, 109.0, 114.0, 124.0, 122.0, 106.0, 110.0, 124.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [58.0, 81.0, 78.0, 82.0, 76.0, 95.0, 104.0, 114.0, 114.0, 147.0, 119.0, 159.0, 165.0, 173.0, 182.0, 167.0, 188.0, 176.0, 167.0, 165.0, 187.0, 162.0, 191.0, 164.0, 181.0, 170.0, 168.0, 172.0, 182.0, 180.0, 164.0, 171.0, 169.0, 154.0, 144.0, 172.0, 173.0, 198.0, 168.0, 210.0, 178.0, 156.0, 174.0, 177.0, 163.0, 172.0, 206.0, 172.0, 184.0, 197.0, 223.0, 153.0, 162.0, 187.0, 173.0, 201.0, 146.0, 152.0, 240.0, 231.0, 192.0, 208.0, 162.0, 210.0, 192.0, 282.0, 232.0, 174.0, 215.0, 186.0, 227.0, 258.0, 202.0, 265.0, 192.0, 216.0, 239.0, 200.0, 265.0, 210.0, 264.0, 231.0, 179.0, 221.0, 234.0, 184.0, 188.0, 206.0, 157.0, 228.0, 217.0, 227.0, 219.0, 233.0, 191.0, 187.0, 214.0, 190.0, 237.0, 168.0, 155.0, 174.0, 165.0, 157.0, 155.0, 136.0, 154.0, 133.0, 124.0, 167.0, 187.0, 158.0, 188.0, 161.0, 168.0, 130.0, 164.0, 109.0, 181.0, 166.0, 146.0, 145.0, 130.0, 132.0, 130.0, 145.0, 125.0, 107.0, 130.0, 147.0, 128.0, 137.0, 149.0, 151.0, 133.0, 117.0, 167.0, 153.0, 134.0, 131.0, 117.0, 116.0, 100.0, 125.0, 121.0, 139.0, 125.0, 139.0, 124.0, 118.0, 103.0, 142.0, 95.0, 127.0, 109.0, 102.0, 110.0, 119.0, 101.0, 129.0, 122.0, 143.0, 119.0, 131.0, 102.0, 117.0, 98.0, 140.0, 129.0, 106.0, 76.0, 115.0, 81.0, 87.0, 118.0, 84.0, 101.0, 118.0, 99.0, 99.0, 107.0, 108.0, 137.0, 131.0, 109.0, 123.0, 107.0, 104.0, 102.0, 138.0, 125.0, 119.0, 91.0, 79.0, 87.0, 112.0, 104.0, 98.0, 101.0, 109.0, 135.0, 98.0, 89.0, 117.0, 106.0, 127.0, 103.0, 111.0, 122.0, 102.0, 92.0, 99.0, 110.0, 93.0, 123.0, 114.0, 133.0, 87.0, 114.0, 121.0, 111.0, 95.0, 93.0, 102.0, 127.0, 88.0, 127.0, 114.0, 107.0, 110.0, 101.0, 110.0, 108.0, 99.0, 106.0, 126.0, 92.0, 96.0, 94.0, 77.0, 124.0, 119.0, 91.0, 105.0, 110.0, 103.0, 97.0, 116.0, 104.0, 97.0, 117.0, 92.0, 110.0, 114.0, 97.0, 101.0, 92.0, 105.0, 93.0, 141.0, 93.0, 106.0, 116.0, 107.0, 122.0, 107.0, 128.0, 100.0, 94.0, 105.0, 124.0, 114.0, 94.0, 80.0, 98.0, 105.0, 97.0, 99.0, 132.0, 94.0, 99.0, 93.0, 108.0, 108.0, 107.0, 111.0, 134.0, 114.0, 104.0, 102.0, 123.0, 108.0, 109.0, 107.0, 110.0, 121.0, 92.0, 94.0, 130.0, 128.0, 130.0, 83.0, 110.0, 130.0, 105.0, 99.0, 106.0, 107.0, 101.0, 100.0, 98.0, 131.0, 101.0, 116.0, 89.0, 106.0, 114.0, 115.0, 112.0, 110.0, 128.0, 92.0, 88.0, 112.0, 108.0, 106.0, 83.0, 113.0, 129.0, 126.0, 99.0, 118.0, 98.0, 101.0, 102.0, 103.0, 119.0, 126.0, 128.0, 110.0, 107.0, 128.0, 125.0, 119.0, 113.0, 89.0, 102.0, 103.0, 126.0, 141.0, 95.0, 106.0, 117.0, 109.0, 93.0, 109.0, 111.0, 138.0, 124.0, 114.0, 106.0, 92.0, 109.0, 105.0, 144.0, 122.0, 108.0, 112.0, 86.0, 100.0, 127.0, 108.0, 100.0, 113.0, 99.0, 103.0, 104.0, 96.0, 125.0, 122.0, 97.0, 128.0, 117.0, 121.0, 133.0, 115.0, 95.0, 126.0, 117.0, 136.0, 118.0, 108.0, 135.0, 109.0, 114.0, 124.0, 122.0, 106.0, 110.0, 124.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95625, 179.95625, 179.95625, 179.95625, 179.95625, 179.95624, 179.95621, 179.95612, 179.95593, 179.95575, 179.95447, 179.9538, 179.95322, 179.95126, 179.95035, 179.94966, 179.94905, 179.94916, 179.94939, 179.94966, 179.9496, 179.94933, 179.94919, 179.94952, 179.95036, 179.95168, 179.95288, 179.95392, 179.95509, 179.9565, 179.95851, 179.96069, 179.963, 179.96532, 179.96788, 179.97118, 179.97482, 179.97873, 179.98279, 179.98714, 179.99208, 179.99753, 180.00325, 180.00955, 180.01634, 180.02382, 180.03171, 180.04016, 180.04951, 180.05969, 180.07059, 180.08221, 180.09441, 180.10721, 180.12059, 180.13457, 180.14899, 180.16373, 180.1792, 180.19586, 180.21344, 180.23199, 180.25226, 180.2733, 180.2948, 180.31709, 180.34032, 180.36464, 180.38991, 180.41573, 180.44231, 180.46947, 180.49721, 180.52528, 180.55406, 180.5829, 180.61168, 180.64125, 180.67117, 180.70154, 180.73244, 180.76378, 180.79633, 180.82928, 180.86198, 180.89581, 180.92958, 180.96359, 180.99808, 181.03401, 181.07187, 181.1104, 181.14795, 181.18536, 181.22249, 181.26071, 181.29898, 181.33658, 181.37422, 181.41164, 181.4467, 181.47968, 181.5123, 181.54552, 181.57919, 181.61421, 181.65012, 181.68695, 181.72267, 181.7587, 181.79526, 181.83344, 181.87288, 181.91354, 181.9543, 181.99518, 182.03568, 182.07515, 182.11353, 182.15218, 182.19164, 182.23108, 182.2708, 182.30989, 182.34795, 182.3871, 182.42479, 182.46089, 182.49536, 182.52867, 182.5638, 182.60063, 182.63989, 182.67992, 182.72049, 182.76151, 182.80296, 182.8448, 182.88582, 182.92665, 182.96825, 183.00778, 183.04619, 183.08208, 183.117, 183.15222, 183.18738, 183.22598, 183.2657, 183.30598, 183.34494, 183.38196, 183.41934, 183.45613, 183.49393, 183.53142, 183.56673, 183.60075, 183.63268, 183.66296, 183.69357, 183.7247, 183.76031, 183.79965, 183.83946, 183.87967, 183.91869, 183.95782, 183.99774, 184.03601, 184.07205, 184.10704, 184.14296, 184.17989, 184.21503, 184.24945, 184.28268, 184.31783, 184.35512, 184.39378, 184.43393, 184.47366, 184.51508, 184.55717, 184.59872, 184.64001, 184.68074, 184.71964, 184.75798, 184.79604, 184.83191, 184.86661, 184.90184, 184.9364, 184.96959, 185.00362, 185.0423, 185.08412, 185.12758, 185.17178, 185.21582, 185.26006, 185.30214, 185.34361, 185.3847, 185.42496, 185.46634, 185.50591, 185.54526, 185.58424, 185.62386, 185.6624, 185.7025, 185.74159, 185.78154, 185.82208, 185.86279, 185.90271, 185.94293, 185.98375, 186.0233, 186.05884, 186.09236, 186.12791, 186.16458, 186.20477, 186.24573, 186.28658, 186.32719, 186.36766, 186.40819, 186.44913, 186.48967, 186.53146, 186.57472, 186.61908, 186.66409, 186.70798, 186.75232, 186.79475, 186.83501, 186.8761, 186.91815, 186.96135, 187.00375, 187.04543, 187.08774, 187.13051, 187.17398, 187.21738, 187.26135, 187.30682, 187.3519, 187.39789, 187.44398, 187.48967, 187.53412, 187.57758, 187.62079, 187.66299, 187.70578, 187.74741, 187.79074, 187.83516, 187.8799, 187.92366, 187.9662, 188.00873, 188.0517, 188.09543, 188.13933, 188.183, 188.2269, 188.2719, 188.31848, 188.36552, 188.41412, 188.46288, 188.51031, 188.55696, 188.60126, 188.64514, 188.68958, 188.7356, 188.78317, 188.82912, 188.87651, 188.92406, 188.97069, 189.0186, 189.06526, 189.11108, 189.15532, 189.20073, 189.24802, 189.29507, 189.3419, 189.38878, 189.43637, 189.48433, 189.53323, 189.58208, 189.63031, 189.67888, 189.72659, 189.7742, 189.82292, 189.87331, 189.92422, 189.97572, 190.02654, 190.07675, 190.12685, 190.17654, 190.22655, 190.27744, 190.32918, 190.38191, 190.43228, 190.48412, 190.53688, 190.58897, 190.6412, 190.69144, 190.74126, 190.79027, 190.84029, 190.89107, 190.94135, 190.99312, 191.04454, 191.09538, 191.14601, 191.19763, 191.25024, 191.3022, 191.35342, 191.40527, 191.45781, 191.51038, 191.56477, 191.61903, 191.67284, 191.72745, 191.78351, 191.83809, 191.89211, 191.94516, 191.99768, 192.0515, 192.10683, 192.16144, 192.21646, 192.27127, 192.3248, 192.37834, 192.43166, 192.48701, 192.54335, 192.59961, 192.65665, 192.71281, 192.76929, 192.82428, 192.88118, 192.93932, 192.99641, 193.05295, 193.10945, 193.16679, 193.22235, 193.27766, 193.33466, 193.38956, 193.44543, 193.4995, 193.55339, 193.60861, 193.66547, 193.72427, 193.78304, 193.84152, 193.8996, 193.95851, 194.01683, 194.07661, 194.13618, 194.19662, 194.25862, 194.32071, 194.3831, 194.44382, 194.50331, 194.56212, 194.62186, 194.67973, 194.73642, 194.7941, 194.85469, 194.91579]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.95625, 179.95625, 179.95625, 179.95625, 179.95625, 179.95624, 179.95621, 179.95612, 179.95593, 179.95575, 179.95447, 179.9538, 179.95322, 179.95126, 179.95035, 179.94966, 179.94905, 179.94916, 179.94939, 179.94966, 179.9496, 179.94933, 179.94919, 179.94952, 179.95036, 179.95168, 179.95288, 179.95392, 179.95509, 179.9565, 179.95851, 179.96069, 179.963, 179.96532, 179.96788, 179.97118, 179.97482, 179.97873, 179.98279, 179.98714, 179.99208, 179.99753, 180.00325, 180.00955, 180.01634, 180.02382, 180.03171, 180.04016, 180.04951, 180.05969, 180.07059, 180.08221, 180.09441, 180.10721, 180.12059, 180.13457, 180.14899, 180.16373, 180.1792, 180.19586, 180.21344, 180.23199, 180.25226, 180.2733, 180.2948, 180.31709, 180.34032, 180.36464, 180.38991, 180.41573, 180.44231, 180.46947, 180.49721, 180.52528, 180.55406, 180.5829, 180.61168, 180.64125, 180.67117, 180.70154, 180.73244, 180.76378, 180.79633, 180.82928, 180.86198, 180.89581, 180.92958, 180.96359, 180.99808, 181.03401, 181.07187, 181.1104, 181.14795, 181.18536, 181.22249, 181.26071, 181.29898, 181.33658, 181.37422, 181.41164, 181.4467, 181.47968, 181.5123, 181.54552, 181.57919, 181.61421, 181.65012, 181.68695, 181.72267, 181.7587, 181.79526, 181.83344, 181.87288, 181.91354, 181.9543, 181.99518, 182.03568, 182.07515, 182.11353, 182.15218, 182.19164, 182.23108, 182.2708, 182.30989, 182.34795, 182.3871, 182.42479, 182.46089, 182.49536, 182.52867, 182.5638, 182.60063, 182.63989, 182.67992, 182.72049, 182.76151, 182.80296, 182.8448, 182.88582, 182.92665, 182.96825, 183.00778, 183.04619, 183.08208, 183.117, 183.15222, 183.18738, 183.22598, 183.2657, 183.30598, 183.34494, 183.38196, 183.41934, 183.45613, 183.49393, 183.53142, 183.56673, 183.60075, 183.63268, 183.66296, 183.69357, 183.7247, 183.76031, 183.79965, 183.83946, 183.87967, 183.91869, 183.95782, 183.99774, 184.03601, 184.07205, 184.10704, 184.14296, 184.17989, 184.21503, 184.24945, 184.28268, 184.31783, 184.35512, 184.39378, 184.43393, 184.47366, 184.51508, 184.55717, 184.59872, 184.64001, 184.68074, 184.71964, 184.75798, 184.79604, 184.83191, 184.86661, 184.90184, 184.9364, 184.96959, 185.00362, 185.0423, 185.08412, 185.12758, 185.17178, 185.21582, 185.26006, 185.30214, 185.34361, 185.3847, 185.42496, 185.46634, 185.50591, 185.54526, 185.58424, 185.62386, 185.6624, 185.7025, 185.74159, 185.78154, 185.82208, 185.86279, 185.90271, 185.94293, 185.98375, 186.0233, 186.05884, 186.09236, 186.12791, 186.16458, 186.20477, 186.24573, 186.28658, 186.32719, 186.36766, 186.40819, 186.44913, 186.48967, 186.53146, 186.57472, 186.61908, 186.66409, 186.70798, 186.75232, 186.79475, 186.83501, 186.8761, 186.91815, 186.96135, 187.00375, 187.04543, 187.08774, 187.13051, 187.17398, 187.21738, 187.26135, 187.30682, 187.3519, 187.39789, 187.44398, 187.48967, 187.53412, 187.57758, 187.62079, 187.66299, 187.70578, 187.74741, 187.79074, 187.83516, 187.8799, 187.92366, 187.9662, 188.00873, 188.0517, 188.09543, 188.13933, 188.183, 188.2269, 188.2719, 188.31848, 188.36552, 188.41412, 188.46288, 188.51031, 188.55696, 188.60126, 188.64514, 188.68958, 188.7356, 188.78317, 188.82912, 188.87651, 188.92406, 188.97069, 189.0186, 189.06526, 189.11108, 189.15532, 189.20073, 189.24802, 189.29507, 189.3419, 189.38878, 189.43637, 189.48433, 189.53323, 189.58208, 189.63031, 189.67888, 189.72659, 189.7742, 189.82292, 189.87331, 189.92422, 189.97572, 190.02654, 190.07675, 190.12685, 190.17654, 190.22655, 190.27744, 190.32918, 190.38191, 190.43228, 190.48412, 190.53688, 190.58897, 190.6412, 190.69144, 190.74126, 190.79027, 190.84029, 190.89107, 190.94135, 190.99312, 191.04454, 191.09538, 191.14601, 191.19763, 191.25024, 191.3022, 191.35342, 191.40527, 191.45781, 191.51038, 191.56477, 191.61903, 191.67284, 191.72745, 191.78351, 191.83809, 191.89211, 191.94516, 191.99768, 192.0515, 192.10683, 192.16144, 192.21646, 192.27127, 192.3248, 192.37834, 192.43166, 192.48701, 192.54335, 192.59961, 192.65665, 192.71281, 192.76929, 192.82428, 192.88118, 192.93932, 192.99641, 193.05295, 193.10945, 193.16679, 193.22235, 193.27766, 193.33466, 193.38956, 193.44543, 193.4995, 193.55339, 193.60861, 193.66547, 193.72427, 193.78304, 193.84152, 193.8996, 193.95851, 194.01683, 194.07661, 194.13618, 194.19662, 194.25862, 194.32071, 194.3831, 194.44382, 194.50331, 194.56212, 194.62186, 194.67973, 194.73642, 194.7941, 194.85469, 194.91579]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [18.78556, 0.6433, 0.64729, 0.63688, 0.63863, 0.64094, 0.6349, 0.97491, 0.63959, 0.63938, 0.63992, 0.63559, 0.63842, 0.63697, 0.63738, 0.64112, 0.63959, 0.64348, 0.63705, 0.6364, 0.63918, 0.63292, 0.6437, 0.64018, 0.639, 0.63548, 0.63416, 0.64052, 0.6394, 0.64087, 0.93505, 0.64011, 0.63922, 0.63683, 0.63698, 0.63707, 0.63678, 0.63951, 0.63884, 0.63971, 0.64127, 0.63397, 0.63425, 0.63678, 0.64689, 0.63996, 0.6373, 0.63968, 0.63439, 0.63168, 0.63761, 0.63699, 0.63824, 0.71804, 0.64031, 0.63865, 0.64029, 0.63765, 0.63483, 0.63106, 0.64044, 0.64084, 0.64009, 0.63302, 0.63552, 0.634, 0.64042, 0.62983, 0.63367, 0.63643, 0.6354, 0.63829, 0.64059, 0.75259, 0.63372, 0.63627, 0.6387, 0.73904, 0.63828, 0.63771, 0.6359, 0.63693, 0.63456, 0.63441, 0.63425, 0.63785, 0.63673, 0.63659, 0.63691, 0.63886, 0.63666, 0.63099, 0.63434, 0.63606, 0.63766, 0.63693, 0.63641, 0.63421, 0.74335, 0.63417, 0.73325, 0.63333, 0.63749, 0.63466, 0.63579, 0.6328, 0.63166, 0.63446, 0.63178, 0.63147, 0.63478, 0.63778, 0.63144, 0.63332, 0.63409, 0.63176, 0.63302, 0.63438, 0.63574, 0.63649, 0.63622, 0.63188, 0.63339, 0.63517, 0.72118, 0.63229, 0.63429, 0.63655, 0.63599, 0.6353, 0.63271, 0.63372, 0.64125, 0.63512, 0.63455, 0.63532, 0.63725, 0.63591, 0.63729, 0.63999, 0.63638, 0.63338, 0.63695, 0.63822, 0.64221, 0.635, 0.63426, 0.63954, 0.63843, 0.75293, 0.63573, 0.63901, 0.63561, 0.63959, 0.6361, 0.63665, 0.64435, 0.63719, 0.63371, 0.63219, 0.6406, 0.64456, 0.63924, 0.635, 0.6327, 0.6352, 0.63564, 0.63957, 0.63877, 0.73034, 0.73934, 0.64019, 0.63815, 0.63937, 0.75337, 0.63669, 0.63936, 0.63737, 0.6461, 0.63756, 0.63312, 0.63542, 0.63878, 0.6388, 0.64047, 0.63637, 0.63586, 0.63666, 0.63721, 0.63734, 0.63786, 0.63594, 0.8184, 0.73163, 0.72764, 0.63564, 0.63408, 0.63622, 0.64045, 0.63686, 0.62364, 0.64914, 0.64308, 0.64069, 0.63927, 0.64269, 0.64288, 0.64533, 0.64376, 0.64236, 0.64125, 0.64212, 0.6369, 0.63583, 0.74464, 0.63698, 0.72591, 0.64074, 0.73419, 0.63849, 0.63726, 0.64412, 0.64282, 0.75083, 0.63592, 0.63941, 0.63766, 0.63791, 0.63977, 0.63509, 0.6399, 0.64297, 0.63884, 0.63671, 0.6435, 0.64374, 0.64843, 0.64579, 0.63861, 0.64594, 0.64077, 0.63925, 0.72846, 0.639, 0.64699, 0.6369, 0.63194, 0.63558, 0.64203, 0.63965, 0.63904, 0.63895, 0.63899, 0.64164, 0.63997, 0.63805, 0.63955, 0.63823, 0.64646, 0.64468, 0.64926, 0.64434, 0.6452, 0.64591, 0.64664, 0.63886, 0.731, 0.64411, 0.64842, 0.6425, 0.64476, 0.63269, 0.63913, 0.63471, 0.63896, 0.63597, 0.63778, 0.63815, 0.6401, 0.64693, 0.64595, 0.64455, 0.64718, 0.64189, 0.63449, 0.75535, 0.6495, 0.6344, 0.63238, 0.64302, 0.6447, 0.64478, 0.63878, 0.63865, 0.64385, 0.64709, 0.64475, 0.63872, 0.63717, 0.64047, 0.64341, 0.6397, 0.64191, 0.63957, 0.63403, 0.64098, 0.64479, 0.64926, 0.74478, 0.73898, 0.64632, 0.64647, 0.63797, 0.64641, 0.64397, 0.64203, 0.645, 0.64045, 0.64179, 0.64038, 0.64201, 0.64156, 0.64501, 0.64116, 0.63858, 0.63331, 0.63441, 0.63583, 0.64119, 0.6353, 0.63464, 0.63359, 0.63663, 0.64109, 0.6316, 0.63418, 0.63702, 0.63806, 0.64097, 0.63561, 0.63886, 0.63666, 0.63662, 0.64007, 0.64226, 0.64759, 0.64499, 0.6441, 0.63331, 0.63366, 0.63388, 0.64218, 0.6449, 0.7739, 0.64344, 0.64344, 0.64738, 0.64398, 0.64107, 0.64511, 0.64245, 0.64068, 0.6375, 0.63653, 0.63463, 0.63795, 0.64039, 0.6391, 0.63754, 0.63814, 0.64098, 0.63698, 0.63569, 0.63797, 0.63695, 0.64036, 0.63449, 0.63592, 0.72519, 0.64273, 0.63744, 0.63929, 0.63719, 0.64021, 0.64007, 0.63925, 0.63833, 0.63918, 0.63915, 0.64067, 0.64172, 0.63687, 0.63877, 0.63737, 0.64309, 0.6455, 0.64316, 0.63731, 0.6383, 0.63962]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60423]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60423]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [271.57376]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [271.57376]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/model_config.yaml
index 2ad08b8d3a..399dbd1c6e 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp1_fp8_no_model_parallel/model_config.yaml
@@ -1,6 +1,7 @@
 ENV_VARS:
   CUDA_DEVICE_MAX_CONNECTIONS: 1
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
 MODEL_ARGS:
@@ -45,7 +46,7 @@ MODEL_ARGS:
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/golden_values.json
new file mode 100644
index 0000000000..e59a5682c9
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [21.16929, 0.69842, 0.69865, 0.68092, 0.68114, 0.68076, 0.68553, 0.6784, 0.70132, 0.68656, 0.68867, 0.69143, 0.69023, 0.68774, 0.70094, 0.68596, 0.68549, 0.6811, 0.68151, 0.67743, 0.6818, 0.67512, 0.68645, 0.67903, 0.68158, 0.68543, 0.68715, 0.68897, 0.70747, 0.68759, 0.68732, 0.68723, 0.69033, 0.68094, 0.68856, 0.6856, 0.69221, 0.68087, 0.69125, 0.68605, 0.69475, 0.68504, 0.6893, 0.69096, 0.69541, 0.70004, 0.69576, 0.69211, 0.70539, 0.69068, 0.68902, 0.69335, 0.68369, 0.68436, 0.68239, 0.68834, 0.6958, 0.68962, 0.68485, 0.69578, 0.6843, 0.68984, 0.69245, 0.68747, 0.68675, 0.69129, 0.68873, 0.68069, 0.69138, 0.69036, 0.68756, 0.68003, 0.68118, 0.68219, 0.68967, 0.68462, 0.68795, 0.68699, 0.6881, 0.6895, 0.6908, 0.68981, 0.68371, 0.68631, 0.68376, 0.81573, 0.69039, 0.69127, 0.69453, 0.69743, 0.69357, 0.68918, 0.68915, 0.68957, 0.69407, 0.68945, 0.69186, 0.68603, 0.68977, 0.70044, 0.69469, 0.69533, 0.69415, 0.69884, 0.69538, 0.69372, 0.69623, 0.69454, 0.6948, 0.69135, 0.69206, 0.68673, 0.68936, 0.68303, 0.68538, 0.68582, 0.69851, 0.70083, 0.69592, 0.69452, 0.69303, 0.69071, 0.70246, 0.6973, 0.69795, 0.69114, 0.69795, 0.69698, 0.69429, 0.69158, 0.69376, 0.69794, 0.69244, 0.69205, 0.69394, 0.69551, 0.69657, 0.69487, 0.69462, 0.69874, 0.69622, 0.69596, 0.69702, 0.69605, 0.69381, 0.68895, 0.69096, 0.69099, 0.69224, 0.68822, 0.69238, 0.68894, 0.69956, 0.69462, 0.69596, 0.69826, 0.69791, 0.69829, 0.69528, 0.69581, 0.69246, 0.69712, 0.69164, 0.69373, 0.69112, 0.69522, 0.68973, 0.69375, 0.69191, 0.69554, 0.69908, 0.69725, 0.69744, 0.69566, 0.69832, 0.69791, 0.69806, 0.69817, 0.69569, 0.69697, 0.69849, 0.69511, 0.69491, 0.69873, 0.69972, 0.70371, 0.69973, 0.70041, 0.69955, 0.69404, 0.69642, 0.69525, 0.70125, 0.69189, 0.70768, 0.71527, 0.70077, 0.69532, 0.6961, 0.7031, 0.67909, 0.68793, 0.70461, 0.69523, 0.69673, 0.70017, 0.69796, 0.69461, 0.70307, 0.69829, 0.69545, 0.69288, 0.75214, 0.70015, 0.70134, 0.69495, 0.70155, 0.70094, 0.69651, 0.69772, 0.69954, 0.69592, 0.6977, 0.69059, 0.69677, 0.69829, 0.69779, 0.69192, 0.69617, 0.69978, 0.68964, 0.69432, 0.69761, 0.69629, 0.69975, 0.69141, 0.69977, 0.69704, 0.70403, 0.68958, 0.69117, 0.68705, 0.69675, 0.68817, 0.69828, 0.69189, 0.69446, 0.6924, 0.69063, 0.691, 0.69163, 0.69402, 0.69605, 0.69383, 0.69327, 0.69636, 0.69175, 0.69468, 0.69281, 0.70044, 0.70067, 0.7016, 0.69557, 0.69614, 0.69761, 0.69793, 0.69322, 0.69689, 0.70043, 0.69446, 0.69543, 0.69346, 0.69441, 0.68931, 0.69592, 0.6914, 0.6929, 0.69539, 0.69954, 0.69999, 0.69447, 0.69508, 0.69638, 0.69699, 0.69614, 0.69655, 0.6957, 0.69348, 0.698, 0.70136, 0.69861, 0.69224, 0.69369, 0.69763, 0.69759, 0.69166, 0.69413, 0.69071, 0.69463, 0.69072, 0.69754, 0.69663, 0.69249, 0.69603, 0.80113, 0.69556, 0.69325, 0.69439, 0.69712, 0.69274, 0.69473, 0.68837, 0.69493, 0.69602, 0.69314, 0.69884, 0.70264, 0.70625, 0.69696, 0.69541, 0.69344, 0.70656, 0.69704, 0.69417, 0.70121, 0.69558, 0.7002, 0.815, 0.69817, 0.69499, 0.70038, 0.70281, 0.70226, 0.69884, 0.69724, 0.69581, 0.69287, 0.69618, 0.71318, 0.69943, 0.70407, 0.69607, 0.69718, 0.68881, 0.69211, 0.69118, 0.69873, 0.69888, 0.70284, 0.6967, 0.70012, 0.69679, 0.69994, 0.69768, 0.7015, 0.70388, 0.69342, 0.69641, 0.70208, 0.6909, 0.69959, 0.69723, 0.69969, 0.70232, 0.69828, 0.697, 0.69714, 0.69676, 0.69506, 0.69683, 0.69519, 0.68973, 0.70075, 0.69457, 0.69842, 0.69584, 0.69872, 0.69358, 0.69875, 0.69346, 0.70004, 0.69971, 0.70151, 0.70016, 0.70414, 0.70754, 0.70082, 0.69723, 0.70207, 0.70466, 0.70276, 0.69824, 0.70085, 0.70049, 0.70134, 0.70037, 0.705, 0.70761, 0.70114, 0.69824]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.92979, 0.36862, 0.36896, 0.35994, 0.3634, 0.36131, 0.37528, 0.36745, 0.39414, 0.37596, 0.3798, 0.38001, 0.38263, 0.37794, 0.39251, 0.3769, 0.37612, 0.3675, 0.37072, 0.36701, 0.37163, 0.36679, 0.37704, 0.36833, 0.37308, 0.37264, 0.37893, 0.37759, 0.39953, 0.37377, 0.37903, 0.37511, 0.37891, 0.37243, 0.38146, 0.37534, 0.38244, 0.37164, 0.38228, 0.37646, 0.38605, 0.37539, 0.38035, 0.38244, 0.38642, 0.3893, 0.38511, 0.3827, 0.39156, 0.3782, 0.37799, 0.38401, 0.37401, 0.37169, 0.37072, 0.37641, 0.38295, 0.38051, 0.37444, 0.38482, 0.37469, 0.38129, 0.38054, 0.37571, 0.37578, 0.37992, 0.37782, 0.37386, 0.3813, 0.38374, 0.3775, 0.37428, 0.37254, 0.37234, 0.37719, 0.37627, 0.37853, 0.37526, 0.38087, 0.38099, 0.38071, 0.38191, 0.37329, 0.3773, 0.3734, 0.5018, 0.38253, 0.38164, 0.38606, 0.38733, 0.38592, 0.38071, 0.37964, 0.37907, 0.38532, 0.37904, 0.38222, 0.37656, 0.38031, 0.38646, 0.38574, 0.38602, 0.37899, 0.38893, 0.38764, 0.38446, 0.38488, 0.38659, 0.38646, 0.38256, 0.38198, 0.37894, 0.38195, 0.37524, 0.37462, 0.37752, 0.38757, 0.39104, 0.38931, 0.38235, 0.38351, 0.38268, 0.39375, 0.3868, 0.38798, 0.38182, 0.39008, 0.38803, 0.38668, 0.38465, 0.38639, 0.38737, 0.38331, 0.37911, 0.38492, 0.38652, 0.38697, 0.38654, 0.38596, 0.39074, 0.38492, 0.38717, 0.38731, 0.38942, 0.386, 0.38148, 0.38444, 0.38374, 0.38416, 0.37792, 0.37748, 0.37957, 0.39104, 0.38581, 0.38566, 0.38678, 0.38966, 0.38882, 0.38683, 0.38264, 0.38507, 0.38712, 0.38306, 0.38289, 0.38103, 0.38363, 0.37743, 0.37875, 0.37956, 0.38316, 0.3891, 0.38796, 0.38596, 0.38565, 0.38554, 0.38556, 0.38505, 0.38092, 0.38387, 0.38393, 0.38859, 0.37887, 0.38497, 0.38623, 0.39043, 0.39246, 0.38914, 0.38962, 0.38901, 0.38336, 0.38644, 0.38387, 0.38958, 0.38133, 0.39066, 0.39461, 0.39129, 0.38237, 0.3862, 0.39181, 0.37212, 0.37912, 0.39389, 0.384, 0.38439, 0.38586, 0.38505, 0.38157, 0.38622, 0.38765, 0.38617, 0.38274, 0.44388, 0.39087, 0.3907, 0.38612, 0.38867, 0.39114, 0.38539, 0.38934, 0.38921, 0.38784, 0.38206, 0.38157, 0.38685, 0.39031, 0.38789, 0.38326, 0.38644, 0.38897, 0.38075, 0.3856, 0.38903, 0.3866, 0.38941, 0.37995, 0.38647, 0.388, 0.3933, 0.38074, 0.38111, 0.37964, 0.38635, 0.37942, 0.38546, 0.38117, 0.38291, 0.38281, 0.38246, 0.38276, 0.38171, 0.382, 0.3865, 0.37957, 0.3856, 0.38543, 0.38204, 0.38551, 0.38485, 0.39262, 0.39183, 0.38966, 0.38778, 0.38805, 0.3857, 0.3903, 0.38332, 0.38621, 0.38966, 0.38839, 0.3794, 0.38725, 0.38481, 0.38106, 0.38522, 0.3806, 0.38384, 0.38521, 0.38656, 0.39255, 0.38382, 0.38686, 0.38703, 0.38844, 0.38459, 0.38745, 0.38311, 0.38465, 0.38785, 0.39146, 0.38846, 0.38178, 0.38121, 0.38932, 0.38613, 0.38272, 0.38328, 0.38309, 0.38433, 0.38086, 0.38574, 0.38715, 0.38325, 0.38613, 0.4565, 0.38631, 0.38538, 0.38553, 0.38639, 0.38282, 0.38384, 0.37918, 0.38658, 0.38666, 0.38487, 0.39121, 0.3908, 0.39786, 0.3849, 0.38844, 0.38522, 0.394, 0.38769, 0.38524, 0.39367, 0.38775, 0.39338, 0.50382, 0.39159, 0.38743, 0.39102, 0.39523, 0.39356, 0.39205, 0.38578, 0.38801, 0.38304, 0.38678, 0.3987, 0.39171, 0.39597, 0.38708, 0.3908, 0.38146, 0.38222, 0.38202, 0.39012, 0.39068, 0.39269, 0.38682, 0.39099, 0.38924, 0.39219, 0.38971, 0.39066, 0.39542, 0.38474, 0.38829, 0.39181, 0.38288, 0.38918, 0.3886, 0.39087, 0.39457, 0.3877, 0.3877, 0.38997, 0.39047, 0.38458, 0.38887, 0.3875, 0.38266, 0.38907, 0.38748, 0.38772, 0.387, 0.38822, 0.38247, 0.39155, 0.38528, 0.39151, 0.39019, 0.39332, 0.39078, 0.3911, 0.39847, 0.3899, 0.39043, 0.39299, 0.39763, 0.39582, 0.39107, 0.39252, 0.39507, 0.39717, 0.3953, 0.40187, 0.40236, 0.39559, 0.39145]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.8012, 0.29387, 0.2986, 0.28406, 0.28522, 0.28969, 0.29061, 0.28796, 0.29063, 0.28667, 0.29358, 0.29506, 0.2922, 0.2852, 0.28989, 0.28483, 0.28642, 0.28342, 0.28232, 0.28136, 0.28422, 0.28036, 0.28492, 0.28314, 0.281, 0.28245, 0.28442, 0.28445, 0.28814, 0.28551, 0.2857, 0.28486, 0.28705, 0.28407, 0.28536, 0.28489, 0.28989, 0.28255, 0.28845, 0.28647, 0.28944, 0.28337, 0.28838, 0.28849, 0.2897, 0.29269, 0.28788, 0.28852, 0.29394, 0.28953, 0.28786, 0.28768, 0.28428, 0.28563, 0.28458, 0.28775, 0.29324, 0.28892, 0.28616, 0.29034, 0.28456, 0.28682, 0.28841, 0.28729, 0.28425, 0.28778, 0.28741, 0.2839, 0.28832, 0.28804, 0.2861, 0.28333, 0.28362, 0.28274, 0.28476, 0.28495, 0.28365, 0.28409, 0.28405, 0.28625, 0.28429, 0.28647, 0.28314, 0.28367, 0.28409, 0.28622, 0.28505, 0.28438, 0.28134, 0.28462, 0.28536, 0.28398, 0.28654, 0.2869, 0.28809, 0.28601, 0.28761, 0.28425, 0.28676, 0.2862, 0.28997, 0.28934, 0.28731, 0.29342, 0.28795, 0.28707, 0.2867, 0.28661, 0.28811, 0.28616, 0.28592, 0.28428, 0.28508, 0.28396, 0.28659, 0.28265, 0.28697, 0.2894, 0.28687, 0.28772, 0.28913, 0.28621, 0.29195, 0.28847, 0.29125, 0.28862, 0.29011, 0.29025, 0.28931, 0.28814, 0.28955, 0.2908, 0.28871, 0.28801, 0.28793, 0.28964, 0.29306, 0.29007, 0.28963, 0.29251, 0.29069, 0.29194, 0.28984, 0.29084, 0.28995, 0.28615, 0.28778, 0.28795, 0.2882, 0.28737, 0.2876, 0.28691, 0.29135, 0.28807, 0.28993, 0.29202, 0.29116, 0.29034, 0.28863, 0.29346, 0.29111, 0.29416, 0.29263, 0.293, 0.29317, 0.2931, 0.28845, 0.288, 0.28664, 0.28885, 0.29051, 0.28976, 0.28937, 0.29252, 0.29727, 0.29583, 0.29602, 0.29658, 0.2931, 0.29603, 0.29621, 0.29395, 0.29259, 0.29542, 0.29412, 0.29939, 0.29634, 0.2902, 0.29267, 0.28896, 0.2887, 0.28951, 0.29196, 0.29075, 0.29727, 0.30019, 0.29535, 0.2896, 0.28882, 0.29318, 0.28687, 0.28581, 0.29387, 0.28979, 0.28852, 0.29025, 0.28988, 0.28996, 0.2906, 0.29127, 0.29091, 0.29027, 0.34386, 0.29092, 0.29145, 0.28886, 0.29332, 0.29127, 0.29064, 0.29054, 0.29117, 0.28886, 0.28689, 0.28524, 0.29113, 0.29077, 0.28956, 0.28788, 0.28875, 0.29066, 0.28696, 0.28828, 0.28986, 0.28975, 0.29179, 0.28765, 0.29054, 0.29018, 0.29236, 0.28513, 0.28796, 0.28625, 0.28988, 0.28486, 0.2901, 0.28715, 0.28807, 0.29103, 0.28636, 0.28731, 0.28709, 0.2878, 0.28863, 0.28922, 0.28858, 0.28861, 0.28721, 0.28911, 0.28891, 0.29009, 0.29181, 0.29183, 0.2921, 0.28906, 0.29246, 0.29132, 0.28922, 0.29183, 0.29154, 0.29016, 0.29033, 0.29069, 0.28941, 0.28627, 0.28999, 0.28617, 0.28792, 0.2909, 0.29099, 0.29284, 0.29202, 0.28998, 0.29186, 0.29297, 0.29177, 0.2896, 0.29112, 0.28824, 0.29124, 0.29518, 0.29288, 0.28876, 0.29026, 0.29318, 0.2932, 0.2894, 0.28931, 0.28848, 0.28934, 0.28881, 0.29144, 0.28798, 0.28986, 0.29212, 0.28958, 0.2898, 0.28969, 0.2893, 0.29213, 0.29, 0.29098, 0.29085, 0.29077, 0.29035, 0.29027, 0.29142, 0.29441, 0.29571, 0.29203, 0.29018, 0.29127, 0.29433, 0.29091, 0.28877, 0.29354, 0.29063, 0.29084, 0.29118, 0.29114, 0.29201, 0.29191, 0.29316, 0.29428, 0.29139, 0.29115, 0.29268, 0.28887, 0.29386, 0.29765, 0.29295, 0.29535, 0.29245, 0.29159, 0.28784, 0.29096, 0.28864, 0.2923, 0.29471, 0.29453, 0.2914, 0.29447, 0.29151, 0.29226, 0.29155, 0.29343, 0.29271, 0.28917, 0.29026, 0.2943, 0.28854, 0.29114, 0.29123, 0.2918, 0.29223, 0.29626, 0.29746, 0.29042, 0.29175, 0.29069, 0.29, 0.2892, 0.28808, 0.29535, 0.28977, 0.29205, 0.29056, 0.29189, 0.2899, 0.28981, 0.2895, 0.2929, 0.29123, 0.29288, 0.29252, 0.29518, 0.29616, 0.29356, 0.29361, 0.29532, 0.29564, 0.29465, 0.29223, 0.29483, 0.29279, 0.29075, 0.29144, 0.29105, 0.29375, 0.28857, 0.288]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.30565, 0.00631, 0.0066, 0.00601, 0.00609, 0.00586, 0.00613, 0.00583, 0.00602, 0.00583, 0.00598, 0.00604, 0.00582, 0.00568, 0.00583, 0.0058, 0.00563, 0.00578, 0.00557, 0.0058, 0.00592, 0.00586, 0.0058, 0.00562, 0.00562, 0.00571, 0.00557, 0.00573, 0.00596, 0.00583, 0.00566, 0.00601, 0.00607, 0.00572, 0.00607, 0.00595, 0.00598, 0.00592, 0.00585, 0.00609, 0.00585, 0.0059, 0.00582, 0.00578, 0.00588, 0.00604, 0.00563, 0.00593, 0.00592, 0.00559, 0.00549, 0.00584, 0.00593, 0.00559, 0.00713, 0.00734, 0.00689, 0.00723, 0.00685, 0.00763, 0.00701, 0.00722, 0.0072, 0.00755, 0.00717, 0.00727, 0.00721, 0.00707, 0.00703, 0.00729, 0.00703, 0.00682, 0.00659, 0.00573, 0.00594, 0.00596, 0.00621, 0.00602, 0.00602, 0.00599, 0.00597, 0.00616, 0.0059, 0.00598, 0.00575, 0.00606, 0.00592, 0.00596, 0.00602, 0.00605, 0.00587, 0.00585, 0.00596, 0.00675, 0.00617, 0.0062, 0.00592, 0.00581, 0.00613, 0.00611, 0.00624, 0.00629, 0.00603, 0.00622, 0.00608, 0.00595, 0.00632, 0.00599, 0.00611, 0.00597, 0.00588, 0.00587, 0.0057, 0.00574, 0.00589, 0.00569, 0.00565, 0.00566, 0.0061, 0.00592, 0.00603, 0.00553, 0.00587, 0.00577, 0.00567, 0.00584, 0.00581, 0.00607, 0.00583, 0.00565, 0.00581, 0.0058, 0.00582, 0.00595, 0.0057, 0.00596, 0.00605, 0.00582, 0.00559, 0.00575, 0.00572, 0.00562, 0.00565, 0.00583, 0.00603, 0.00568, 0.00564, 0.00603, 0.00593, 0.0059, 0.00581, 0.0055, 0.00598, 0.00604, 0.00607, 0.00585, 0.00585, 0.00603, 0.00588, 0.00599, 0.00567, 0.00593, 0.00614, 0.0058, 0.00592, 0.00575, 0.00581, 0.00624, 0.00582, 0.00616, 0.00572, 0.00591, 0.0061, 0.00614, 0.00597, 0.00606, 0.00588, 0.00578, 0.00631, 0.00589, 0.00584, 0.00574, 0.00613, 0.00566, 0.0061, 0.00599, 0.0059, 0.00589, 0.00595, 0.00596, 0.00595, 0.00595, 0.00613, 0.00585, 0.00569, 0.00609, 0.00603, 0.00615, 0.00617, 0.00606, 0.06212, 0.00708, 0.00731, 0.00708, 0.00688, 0.0068, 0.00715, 0.00694, 0.00689, 0.00682, 0.00592, 0.00599, 0.00671, 0.00709, 0.00695, 0.00727, 0.00736, 0.00727, 0.00737, 0.00678, 0.00708, 0.00694, 0.00721, 0.00727, 0.00742, 0.00681, 0.00707, 0.00694, 0.00708, 0.00695, 0.00706, 0.00698, 0.00707, 0.0067, 0.00718, 0.00733, 0.00718, 0.00687, 0.00725, 0.00712, 0.00718, 0.00685, 0.00603, 0.00744, 0.00676, 0.00683, 0.00724, 0.00706, 0.00733, 0.00734, 0.00681, 0.00744, 0.00713, 0.00687, 0.00667, 0.00687, 0.00723, 0.00685, 0.00677, 0.00724, 0.00676, 0.00673, 0.0071, 0.00721, 0.00713, 0.00707, 0.00719, 0.00656, 0.00681, 0.0069, 0.00711, 0.00704, 0.00728, 0.00686, 0.00705, 0.00647, 0.00678, 0.00724, 0.00671, 0.00729, 0.00729, 0.00693, 0.00727, 0.00705, 0.0073, 0.0069, 0.00703, 0.00703, 0.00673, 0.00641, 0.00649, 0.0059, 0.00591, 0.00589, 0.00611, 0.00602, 0.00581, 0.00591, 0.006, 0.00615, 0.00591, 0.00611, 0.00606, 0.00605, 0.00645, 0.00595, 0.00594, 0.00596, 0.006, 0.00598, 0.00594, 0.00601, 0.00655, 0.00617, 0.00603, 0.0059, 0.00628, 0.00583, 0.00608, 0.00585, 0.00604, 0.00603, 0.00594, 0.00582, 0.00576, 0.00596, 0.00605, 0.00641, 0.00601, 0.00602, 0.0061, 0.00618, 0.00595, 0.00602, 0.00597, 0.00581, 0.00598, 0.00598, 0.00614, 0.00599, 0.00582, 0.00612, 0.00597, 0.00575, 0.00572, 0.00623, 0.00601, 0.00597, 0.00619, 0.00626, 0.00606, 0.00592, 0.00607, 0.00584, 0.00593, 0.00602, 0.00617, 0.00621, 0.00612, 0.00602, 0.00597, 0.00594, 0.00615, 0.00599, 0.00604, 0.00617, 0.00631, 0.00558, 0.00552, 0.0057, 0.00568, 0.00594, 0.00614, 0.00588, 0.006, 0.00605, 0.00607, 0.00624, 0.00636, 0.00582, 0.00604, 0.00595, 0.0061, 0.00615, 0.00599, 0.00599, 0.00621, 0.00604, 0.00599, 0.00599, 0.00589, 0.00621, 0.00584, 0.00586, 0.00593, 0.00614, 0.00623, 0.00591, 0.00632, 0.00604]}, "forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.95821, 0.02363, 0.0227, 0.02332, 0.02256, 0.02319, 0.0228, 0.02261, 0.0228, 0.02242, 0.02284, 0.02259, 0.02245, 0.02309, 0.02332, 0.02185, 0.02227, 0.02241, 0.02251, 0.02246, 0.02257, 0.02259, 0.02212, 0.02254, 0.02299, 0.02339, 0.02258, 0.02339, 0.02279, 0.02234, 0.0221, 0.02333, 0.02239, 0.02203, 0.02184, 0.02211, 0.02224, 0.022, 0.0223, 0.02282, 0.02196, 0.02285, 0.02194, 0.02233, 0.02238, 0.0221, 0.02287, 0.02259, 0.02353, 0.02258, 0.02174, 0.02244, 0.02248, 0.02249, 0.02286, 0.02274, 0.02231, 0.02301, 0.02252, 0.02226, 0.02309, 0.0226, 0.02248, 0.02257, 0.02247, 0.02239, 0.02245, 0.02239, 0.02245, 0.02226, 0.02251, 0.02235, 0.02229, 0.02229, 0.02224, 0.02218, 0.02269, 0.02222, 0.02297, 0.0233, 0.02355, 0.02353, 0.02351, 0.02353, 0.0231, 0.02266, 0.02205, 0.02248, 0.02239, 0.02243, 0.02337, 0.02243, 0.02265, 0.02251, 0.0227, 0.02251, 0.02262, 0.0223, 0.02239, 0.02302, 0.02253, 0.0224, 0.02341, 0.02267, 0.02201, 0.02288, 0.02223, 0.02234, 0.02247, 0.02274, 0.0227, 0.02223, 0.02278, 0.02249, 0.02233, 0.02353, 0.02284, 0.02293, 0.02146, 0.02395, 0.02287, 0.02228, 0.02286, 0.02372, 0.02285, 0.02195, 0.02251, 0.02292, 0.02278, 0.02298, 0.02247, 0.02293, 0.02269, 0.02272, 0.02289, 0.0229, 0.0226, 0.02277, 0.02291, 0.02243, 0.02298, 0.02242, 0.02233, 0.02273, 0.0224, 0.02231, 0.02213, 0.02282, 0.02271, 0.02257, 0.02245, 0.02266, 0.02226, 0.02234, 0.02242, 0.02287, 0.02231, 0.02272, 0.02271, 0.02261, 0.02279, 0.02239, 0.02238, 0.02237, 0.02245, 0.02246, 0.023, 0.02279, 0.02277, 0.02299, 0.02326, 0.0223, 0.02341, 0.02259, 0.02308, 0.02252, 0.02308, 0.02263, 0.02343, 0.02234, 0.02287, 0.02253, 0.02261, 0.02291, 0.02258, 0.02266, 0.02272, 0.02323, 0.02251, 0.02228, 0.0226, 0.02245, 0.02282, 0.02319, 0.02275, 0.02246, 0.02327, 0.02259, 0.02253, 0.0224, 0.01758, 0.02244, 0.02255, 0.02222, 0.02295, 0.02246, 0.02236, 0.02202, 0.02348, 0.02237, 0.02232, 0.02231, 0.02262, 0.02284, 0.02278, 0.02292, 0.02249, 0.02264, 0.02288, 0.02264, 0.02232, 0.02331, 0.02235, 0.02266, 0.02272, 0.02229, 0.02285, 0.02276, 0.02283, 0.02355, 0.02243, 0.02224, 0.02272, 0.02285, 0.02224, 0.02355, 0.02275, 0.02246, 0.02254, 0.02335, 0.02272, 0.02208, 0.02249, 0.02229, 0.02237, 0.02251, 0.0228, 0.02259, 0.02238, 0.02269, 0.02278, 0.02234, 0.02262, 0.02237, 0.02265, 0.02234, 0.0239, 0.02204, 0.02217, 0.02222, 0.02262, 0.02231, 0.02208, 0.02252, 0.02267, 0.02293, 0.02253, 0.02228, 0.02237, 0.02246, 0.02294, 0.02246, 0.02182, 0.0225, 0.02229, 0.02265, 0.02222, 0.02222, 0.02264, 0.02241, 0.02246, 0.02208, 0.02243, 0.0227, 0.02237, 0.02231, 0.02228, 0.02312, 0.02228, 0.02236, 0.02245, 0.02239, 0.02316, 0.02216, 0.02227, 0.02241, 0.0226, 0.02206, 0.02266, 0.0223, 0.02225, 0.02286, 0.0223, 0.02201, 0.02235, 0.02378, 0.02224, 0.02326, 0.02229, 0.02293, 0.02211, 0.02198, 0.02233, 0.0224, 0.02212, 0.02248, 0.02253, 0.02253, 0.02258, 0.02203, 0.02237, 0.02274, 0.0222, 0.02237, 0.02238, 0.02242, 0.02229, 0.02263, 0.02196, 0.02243, 0.02239, 0.02243, 0.02221, 0.02264, 0.02264, 0.02249, 0.02235, 0.0226, 0.02289, 0.02232, 0.0227, 0.02252, 0.02225, 0.02254, 0.02223, 0.02268, 0.02244, 0.02292, 0.02284, 0.02271, 0.02275, 0.02258, 0.02303, 0.02263, 0.02297, 0.02275, 0.0227, 0.023, 0.02298, 0.02297, 0.02199, 0.02326, 0.02298, 0.02263, 0.02262, 0.02296, 0.02268, 0.0225, 0.02268, 0.02273, 0.02239, 0.02231, 0.02302, 0.02284, 0.02258, 0.02376, 0.02298, 0.02258, 0.02269, 0.02282, 0.02248, 0.02296, 0.02259, 0.02303, 0.02252, 0.02322, 0.02265, 0.0226, 0.02282, 0.0227, 0.02325, 0.02263, 0.02282, 0.02297, 0.02259, 0.02313, 0.02262, 0.02287, 0.02288, 0.02356]}, "forward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3.00337, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00016, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00017, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00015, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00013, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00015, 0.00013, 0.00014, 0.00014, 0.00015, 0.00014, 0.00013, 0.00014, 0.00015, 0.00014, 0.00016, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00015, 0.00015, 0.00014, 0.00016, 0.00013, 0.00016, 0.00014, 0.00015, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00016, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00018, 0.00014, 0.00015, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00015, 0.00013, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00017, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00015, 0.00014, 0.00017, 0.00014, 0.00015, 0.00014, 0.00014, 0.00013, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00018, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00013, 0.00014, 0.00015, 0.00016, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00015, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00016, 0.00014, 0.00014, 0.00015, 0.00015, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00016, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00015, 0.00015, 0.00014, 0.00016, 0.00014, 0.00015, 0.00015, 0.00015]}, "backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02248, 0.02331, 0.02263, 0.02336, 0.02223, 0.02299, 0.02211, 0.02247, 0.0226, 0.02292, 0.02307, 0.02276, 0.02341, 0.02329, 0.02311, 0.02274, 0.02235, 0.0235, 0.02241, 0.02254, 0.0226, 0.02238, 0.02202, 0.02262, 0.02257, 0.02202, 0.02244, 0.02212, 0.02257, 0.02222, 0.02301, 0.02231, 0.02146, 0.02328, 0.0228, 0.02276, 0.02277, 0.02305, 0.02315, 0.02206, 0.02273, 0.02196, 0.02292, 0.0229, 0.02318, 0.02404, 0.02342, 0.02372, 0.024, 0.02283, 0.02293, 0.02329, 0.02241, 0.02288, 0.02249, 0.02209, 0.0225, 0.02317, 0.02289, 0.02337, 0.02275, 0.02241, 0.02374, 0.02164, 0.02208, 0.02228, 0.02281, 0.02282, 0.02272, 0.0226, 0.0227, 0.02228, 0.02281, 0.02266, 0.02389, 0.02245, 0.02241, 0.02233, 0.02295, 0.02231, 0.0221, 0.02223, 0.0226, 0.02234, 0.02195, 0.02202, 0.02245, 0.0226, 0.02275, 0.02248, 0.0222, 0.02241, 0.02244, 0.02231, 0.02257, 0.02222, 0.02266, 0.02423, 0.02272, 0.02227, 0.02299, 0.02249, 0.0224, 0.02471, 0.02315, 0.02261, 0.02228, 0.02296, 0.02277, 0.02251, 0.02275, 0.02249, 0.02349, 0.022, 0.02327, 0.0234, 0.02263, 0.02233, 0.02301, 0.02227, 0.02246, 0.02257, 0.02278, 0.02253, 0.02246, 0.02297, 0.02258, 0.02373, 0.02268, 0.02299, 0.02323, 0.02295, 0.02269, 0.02271, 0.02329, 0.02248, 0.02289, 0.02291, 0.02254, 0.02282, 0.02401, 0.02262, 0.02444, 0.02261, 0.0226, 0.02263, 0.02259, 0.02307, 0.02224, 0.02211, 0.02289, 0.02273, 0.02385, 0.02337, 0.02258, 0.02316, 0.02269, 0.02287, 0.02301, 0.0225, 0.02248, 0.02339, 0.02296, 0.02226, 0.02308, 0.02301, 0.02193, 0.02223, 0.02389, 0.02273, 0.02314, 0.0224, 0.02271, 0.02292, 0.0234, 0.02311, 0.02278, 0.02281, 0.02287, 0.02271, 0.02258, 0.02224, 0.02289, 0.02216, 0.02306, 0.02215, 0.02293, 0.02325, 0.02272, 0.02257, 0.02265, 0.02257, 0.02237, 0.02338, 0.02396, 0.02264, 0.02255, 0.02263, 0.02261, 0.02319, 0.02273, 0.0227, 0.02359, 0.02237, 0.02352, 0.02453, 0.02244, 0.02254, 0.02341, 0.02295, 0.02318, 0.02233, 0.02248, 0.02304, 0.02424, 0.02304, 0.02275, 0.02374, 0.02258, 0.02316, 0.02275, 0.02259, 0.02278, 0.02276, 0.02303, 0.02314, 0.02359, 0.02289, 0.02295, 0.02301, 0.02271, 0.02295, 0.02286, 0.02295, 0.02288, 0.02247, 0.02599, 0.02329, 0.02375, 0.02231, 0.0227, 0.0222, 0.02287, 0.02291, 0.02232, 0.02287, 0.02269, 0.0222, 0.02306, 0.02281, 0.0228, 0.02143, 0.02285, 0.02337, 0.02236, 0.02228, 0.02243, 0.02313, 0.02393, 0.02356, 0.02319, 0.02319, 0.02354, 0.02282, 0.02254, 0.02335, 0.02225, 0.02305, 0.0231, 0.02313, 0.02277, 0.02351, 0.02342, 0.02326, 0.02253, 0.02222, 0.02252, 0.02264, 0.02318, 0.02321, 0.02292, 0.02334, 0.02285, 0.02282, 0.02307, 0.02259, 0.02166, 0.02265, 0.02214, 0.02373, 0.02309, 0.0232, 0.02261, 0.02274, 0.02256, 0.02221, 0.02164, 0.02324, 0.02299, 0.02313, 0.02404, 0.02301, 0.02264, 0.02252, 0.02325, 0.02343, 0.02291, 0.02247, 0.0231, 0.02252, 0.02239, 0.02337, 0.02232, 0.02332, 0.02306, 0.02293, 0.02287, 0.02295, 0.02297, 0.02351, 0.02268, 0.02263, 0.02425, 0.02263, 0.02361, 0.023, 0.02223, 0.02273, 0.02318, 0.02333, 0.0232, 0.02407, 0.02312, 0.0227, 0.02288, 0.02285, 0.02227, 0.0233, 0.02303, 0.02288, 0.0233, 0.0231, 0.02299, 0.02245, 0.02284, 0.02224, 0.02277, 0.02352, 0.02304, 0.02289, 0.02369, 0.02293, 0.02308, 0.02248, 0.02362, 0.02358, 0.02328, 0.02302, 0.0234, 0.02273, 0.02296, 0.02329, 0.0228, 0.0234, 0.02231, 0.02262, 0.02265, 0.02299, 0.02199, 0.02303, 0.02291, 0.02278, 0.02341, 0.0232, 0.02291, 0.02339, 0.02355, 0.02363, 0.02324, 0.02236, 0.023, 0.02327, 0.02343, 0.02262, 0.02317, 0.02371, 0.02282, 0.02307, 0.0239, 0.02366, 0.02297, 0.02286, 0.02285, 0.0232, 0.02342, 0.02385, 0.02348, 0.02254, 0.02321, 0.02256]}, "backward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00014, 0.00018, 0.00017, 0.00019, 0.00013, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00017, 0.00015, 0.00016, 0.00015, 0.00015, 0.00017, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00017, 0.00016, 0.00015, 0.00015, 0.00016, 0.00014, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00019, 0.00015, 0.00015, 0.00017, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00014, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00016, 0.00017, 0.00016, 0.00012, 0.00016, 0.00012, 0.00012, 0.00013, 0.00013, 0.00016, 0.00013, 0.00013, 0.00014, 0.00013, 0.00012, 0.00012, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00017, 0.00014, 0.00017, 0.00013, 0.00013, 0.00013, 0.00019, 0.00014, 0.00014, 0.00013, 0.00018, 0.00013, 0.00014, 0.00013, 0.00016, 0.00015, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00014, 0.00015, 0.00013, 0.00014, 0.00012, 0.00013, 0.00013, 0.00012, 0.00016, 0.00017, 0.00013, 0.00014, 0.00013, 0.00015, 0.00013, 0.00013, 0.00015, 0.00016, 0.00013, 0.00013, 0.00013, 0.00015, 0.00013, 0.00016, 0.00015, 0.00013, 0.00013, 0.00013, 0.00013, 0.00017, 0.00015, 0.00017, 0.00014, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00015, 0.00014, 0.00013, 0.00015, 0.00014, 0.00012, 0.00014, 0.00013, 0.00016, 0.00015, 0.00015, 0.00013, 0.00013, 0.00013, 0.00013, 0.00016, 0.00012, 0.00013, 0.00015, 0.00013, 0.00015, 0.00014, 0.00016, 0.00013, 0.00013, 0.00015, 0.00016, 0.00012, 0.00016, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00019, 0.00013, 0.00013, 0.00014, 0.00015, 0.00014, 0.00013, 0.00014, 0.00016, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00015, 0.00013, 0.00013, 0.00014, 0.00014, 0.00013, 0.00016, 0.00013, 0.00018, 0.00012, 0.00014, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00014, 0.00016, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00018, 0.00013, 0.00013, 0.00013, 0.00014, 0.00015, 0.00014, 0.00014, 0.00012, 0.00013, 0.00013, 0.00014, 0.00014, 0.00015, 0.00015, 0.00013, 0.00013, 0.00014, 0.00013, 0.00012, 0.00013, 0.00012, 0.00015, 0.00013, 0.00013, 0.00014, 0.00015, 0.00012, 0.00013, 0.00013, 0.00014, 0.00013, 0.00012, 0.00017, 0.00013, 0.00015, 0.00017, 0.00013, 0.00014, 0.00016, 0.00012, 0.00014, 0.00013, 0.00014, 0.00013, 0.00015, 0.00015, 0.00016, 0.00017, 0.00013, 0.00018, 0.00014, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00014, 0.00016, 0.00014, 0.00013, 0.00014, 0.00012, 0.00013, 0.00012, 0.00016, 0.00012, 0.00015, 0.00013, 0.00013, 0.00013, 0.00012, 0.00016, 0.00017, 0.00013, 0.00013, 0.00013, 0.00014, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00014, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00014, 0.00015, 0.00013, 0.00013, 0.00013, 0.00017, 0.00014, 0.00014, 0.00016, 0.00013, 0.00015, 0.00014, 0.00017, 0.00016, 0.00014, 0.00014, 0.00013, 0.00015, 0.00012, 0.00013, 0.00012, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00012, 0.00013, 0.00015, 0.00014, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00014, 0.00013, 0.00013, 0.00013, 0.00012, 0.00015, 0.00016, 0.00013, 0.00013, 0.00014, 0.00014, 0.00017, 0.00012, 0.00015, 0.00016, 0.00016, 0.00013, 0.00015, 0.00014, 0.00013, 0.00013, 0.00012, 0.00012, 0.00017, 0.00013, 0.00013, 0.00012, 0.00012]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [7.29163, 0.07663, 0.08035, 0.06332, 0.06621, 0.06965, 0.06672, 0.06872, 0.07455, 0.0683, 0.06975, 0.07264, 0.07308, 0.06869, 0.0749, 0.06785, 0.06696, 0.07011, 0.07008, 0.06771, 0.06763, 0.06853, 0.06929, 0.06793, 0.0646, 0.06794, 0.06582, 0.06618, 0.07898, 0.06585, 0.0677, 0.06681, 0.07017, 0.06602, 0.06883, 0.06722, 0.06997, 0.06853, 0.07057, 0.06872, 0.06884, 0.06699, 0.06869, 0.07012, 0.06782, 0.06999, 0.06845, 0.06563, 0.07187, 0.06575, 0.06637, 0.06468, 0.06438, 0.06646, 0.06395, 0.06524, 0.08025, 0.06764, 0.06976, 0.06968, 0.06431, 0.06784, 0.06839, 0.06965, 0.06878, 0.06848, 0.06691, 0.06998, 0.07092, 0.06857, 0.0693, 0.06815, 0.07095, 0.07046, 0.07279, 0.07009, 0.07045, 0.07242, 0.06971, 0.06878, 0.0711, 0.06854, 0.0703, 0.07136, 0.07206, 0.19699, 0.06856, 0.07017, 0.0772, 0.07413, 0.06965, 0.06662, 0.06863, 0.07002, 0.06852, 0.06895, 0.06723, 0.06766, 0.06739, 0.07615, 0.06865, 0.0659, 0.07051, 0.0678, 0.06754, 0.06717, 0.07145, 0.07015, 0.06808, 0.06744, 0.06521, 0.06518, 0.06265, 0.06299, 0.06279, 0.06454, 0.07004, 0.06844, 0.06842, 0.06744, 0.06305, 0.06615, 0.07084, 0.06889, 0.06934, 0.0652, 0.07021, 0.0665, 0.06497, 0.06458, 0.06483, 0.0654, 0.0651, 0.06488, 0.06369, 0.06434, 0.06672, 0.06482, 0.06827, 0.06829, 0.0643, 0.06825, 0.06762, 0.06752, 0.06536, 0.06267, 0.06412, 0.06238, 0.0644, 0.06315, 0.06427, 0.06278, 0.06772, 0.06453, 0.06547, 0.06433, 0.06477, 0.06262, 0.06246, 0.0656, 0.06412, 0.06447, 0.06356, 0.06614, 0.0655, 0.06558, 0.06542, 0.06499, 0.06312, 0.06403, 0.06715, 0.06427, 0.06479, 0.06361, 0.06722, 0.06583, 0.06476, 0.06651, 0.06877, 0.06755, 0.06567, 0.06624, 0.06526, 0.06717, 0.06755, 0.06946, 0.06655, 0.06526, 0.06418, 0.06359, 0.06533, 0.06548, 0.06698, 0.06537, 0.06464, 0.07565, 0.06673, 0.06462, 0.06523, 0.06525, 0.05829, 0.06037, 0.06399, 0.06429, 0.06234, 0.06138, 0.06591, 0.06529, 0.06565, 0.06508, 0.0686, 0.06838, 0.12228, 0.06666, 0.06636, 0.0641, 0.06601, 0.06468, 0.06395, 0.06568, 0.06779, 0.06425, 0.06928, 0.06612, 0.06928, 0.0652, 0.06359, 0.06153, 0.06449, 0.06439, 0.06432, 0.06445, 0.06351, 0.06481, 0.06503, 0.06334, 0.0646, 0.06418, 0.06493, 0.06414, 0.06257, 0.06426, 0.06752, 0.06251, 0.06434, 0.06117, 0.06509, 0.06177, 0.06484, 0.06385, 0.06538, 0.06711, 0.0659, 0.06606, 0.06549, 0.06518, 0.06537, 0.06313, 0.0654, 0.0676, 0.06603, 0.06663, 0.06705, 0.06676, 0.0651, 0.0677, 0.06421, 0.06506, 0.06513, 0.06577, 0.06915, 0.06804, 0.06617, 0.06569, 0.06722, 0.06636, 0.06674, 0.06574, 0.06698, 0.06664, 0.06663, 0.06459, 0.06384, 0.06515, 0.06699, 0.06757, 0.06645, 0.06668, 0.0657, 0.06812, 0.06673, 0.06651, 0.06468, 0.06953, 0.06688, 0.06585, 0.06531, 0.06508, 0.06559, 0.06487, 0.0647, 0.06539, 0.06861, 0.06738, 0.06026, 0.06597, 0.06493, 0.06467, 0.06738, 0.06641, 0.06506, 0.0673, 0.06795, 0.06714, 0.06848, 0.06828, 0.07103, 0.0742, 0.06691, 0.06638, 0.06521, 0.06791, 0.06493, 0.06647, 0.06851, 0.06674, 0.06949, 0.18067, 0.06896, 0.0653, 0.06795, 0.06966, 0.06981, 0.0677, 0.06607, 0.06924, 0.06499, 0.06831, 0.06832, 0.06949, 0.07135, 0.06537, 0.07037, 0.06461, 0.06603, 0.06572, 0.06904, 0.06866, 0.06911, 0.06296, 0.0684, 0.06727, 0.06737, 0.069, 0.06738, 0.07025, 0.06407, 0.06509, 0.06963, 0.06441, 0.07069, 0.07222, 0.07463, 0.07367, 0.07032, 0.07129, 0.07156, 0.07253, 0.06858, 0.06926, 0.06916, 0.06788, 0.06771, 0.06859, 0.06745, 0.07278, 0.06943, 0.06671, 0.0691, 0.06585, 0.06975, 0.07019, 0.07413, 0.0711, 0.07228, 0.07684, 0.07091, 0.0736, 0.07134, 0.07497, 0.07213, 0.06976, 0.07166, 0.0746, 0.0763, 0.06965, 0.07059, 0.07384, 0.07021, 0.07072]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.84189, 0.0034, 0.00335, 0.0028, 0.00275, 0.0029, 0.00298, 0.00297, 0.00304, 0.00306, 0.00309, 0.00308, 0.00301, 0.00302, 0.00299, 0.00294, 0.003, 0.00307, 0.0031, 0.00304, 0.00303, 0.00294, 0.00305, 0.00298, 0.00301, 0.00306, 0.0029, 0.00302, 0.00303, 0.0031, 0.00306, 0.00304, 0.00303, 0.00301, 0.00294, 0.00305, 0.00312, 0.00303, 0.00301, 0.00328, 0.00302, 0.00288, 0.00306, 0.00304, 0.00304, 0.00303, 0.00299, 0.00297, 0.003, 0.00305, 0.00302, 0.00306, 0.00303, 0.00307, 0.00305, 0.00294, 0.00385, 0.00305, 0.00293, 0.00307, 0.00295, 0.003, 0.00297, 0.00308, 0.00305, 0.00303, 0.00302, 0.00254, 0.00275, 0.00284, 0.00252, 0.00253, 0.00257, 0.00262, 0.00255, 0.00266, 0.00264, 0.0026, 0.00255, 0.00265, 0.00267, 0.00266, 0.00269, 0.0026, 0.00263, 0.00301, 0.00264, 0.00265, 0.00269, 0.00261, 0.00267, 0.00257, 0.00268, 0.0027, 0.00261, 0.00268, 0.00261, 0.00264, 0.00255, 0.00261, 0.00281, 0.00269, 0.00271, 0.00271, 0.00264, 0.00265, 0.00268, 0.0026, 0.00262, 0.00283, 0.00271, 0.00272, 0.00266, 0.00257, 0.00253, 0.00256, 0.00276, 0.00272, 0.00264, 0.00283, 0.00271, 0.00262, 0.00269, 0.00277, 0.00266, 0.0026, 0.00277, 0.00282, 0.00271, 0.00264, 0.00273, 0.00268, 0.00264, 0.00266, 0.0027, 0.00274, 0.00274, 0.0027, 0.00271, 0.00273, 0.00279, 0.0027, 0.00276, 0.00265, 0.0028, 0.00278, 0.00273, 0.00287, 0.00273, 0.00277, 0.00273, 0.00265, 0.00272, 0.00267, 0.00277, 0.00265, 0.00267, 0.0027, 0.00268, 0.00269, 0.00264, 0.00278, 0.00271, 0.00267, 0.00258, 0.00265, 0.00262, 0.00273, 0.00273, 0.00285, 0.00277, 0.00264, 0.00285, 0.00276, 0.00269, 0.00275, 0.00339, 0.00271, 0.00288, 0.00276, 0.00282, 0.00266, 0.00281, 0.00268, 0.00277, 0.00269, 0.00271, 0.0028, 0.00273, 0.00293, 0.00264, 0.00265, 0.00285, 0.0026, 0.00269, 0.00287, 0.00272, 0.00278, 0.0028, 0.00271, 0.00259, 0.00259, 0.00273, 0.00266, 0.0027, 0.00278, 0.00275, 0.0029, 0.00268, 0.00277, 0.0027, 0.00273, 0.00744, 0.00272, 0.00261, 0.00274, 0.00281, 0.00282, 0.00277, 0.00264, 0.00277, 0.00268, 0.00266, 0.00256, 0.00267, 0.00276, 0.00287, 0.00271, 0.00271, 0.00265, 0.00268, 0.00304, 0.00294, 0.00305, 0.0029, 0.00293, 0.00278, 0.00294, 0.00291, 0.00285, 0.00291, 0.00286, 0.00284, 0.00295, 0.0029, 0.0029, 0.00287, 0.00287, 0.0029, 0.00282, 0.00289, 0.0028, 0.0029, 0.00288, 0.0028, 0.00266, 0.0026, 0.00273, 0.00266, 0.00275, 0.00276, 0.00275, 0.00283, 0.0027, 0.00268, 0.00279, 0.00265, 0.00277, 0.00279, 0.00278, 0.00276, 0.00273, 0.00266, 0.00264, 0.00265, 0.00264, 0.00268, 0.00279, 0.00284, 0.00276, 0.00269, 0.00277, 0.00277, 0.00268, 0.00268, 0.00266, 0.00263, 0.00274, 0.0026, 0.00268, 0.00269, 0.00259, 0.00258, 0.00283, 0.00267, 0.00256, 0.00279, 0.0026, 0.00276, 0.00258, 0.00269, 0.00264, 0.00266, 0.00272, 0.10829, 0.00271, 0.00273, 0.00261, 0.00278, 0.00265, 0.00268, 0.00259, 0.00272, 0.00286, 0.00273, 0.00271, 0.00286, 0.00269, 0.00267, 0.0027, 0.00281, 0.0027, 0.00267, 0.00273, 0.0027, 0.00257, 0.0026, 0.00298, 0.0026, 0.00269, 0.00264, 0.00279, 0.00281, 0.00269, 0.0031, 0.0027, 0.0027, 0.00273, 0.0028, 0.00277, 0.00279, 0.00274, 0.00279, 0.00256, 0.00277, 0.00273, 0.00275, 0.00268, 0.00277, 0.00282, 0.0028, 0.00268, 0.00285, 0.00263, 0.00275, 0.00272, 0.0027, 0.00272, 0.00269, 0.00263, 0.00272, 0.00262, 0.00268, 0.0027, 0.00275, 0.0027, 0.00256, 0.00261, 0.00265, 0.00271, 0.00266, 0.00266, 0.00275, 0.00281, 0.00274, 0.00263, 0.00267, 0.00277, 0.00271, 0.00263, 0.00267, 0.00269, 0.00285, 0.00267, 0.00275, 0.00276, 0.00277, 0.0026, 0.00277, 0.0027, 0.00279, 0.00284, 0.00284, 0.0028, 0.00331, 0.00286, 0.0027, 0.00271, 0.00257, 0.00255]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 3e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00071, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00047, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00049, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00047, 0.00045, 0.00046, 0.00046, 0.00045, 0.00047, 0.00045, 0.00046, 0.00046, 0.00046, 0.00048, 0.00046, 0.00048, 0.00045, 0.00046, 0.00048, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00047, 0.00048, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00047, 0.00044, 0.00044, 0.00045, 0.00045, 0.00047, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00049, 0.00045, 0.00046, 0.00044, 0.00046, 0.00047, 0.00045, 0.00045, 0.00045, 0.00046, 0.00047, 0.00045, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00081, 0.00046, 0.00046, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00048, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00046, 0.00047, 0.00046, 0.00047, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00044, 0.00046, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00049, 0.00047, 0.00045, 0.00045, 0.00049, 0.00045, 0.00049, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00049, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00048, 0.00045, 0.00046, 0.00046, 0.00044, 0.00045, 0.00045, 0.00044, 0.00046, 0.00045, 0.00044, 0.00048, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00046, 0.00048, 0.00045, 0.00045, 0.00045, 0.00047, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00051, 0.00049, 0.00045, 0.00046, 0.00047, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00047, 0.00049, 0.0005, 0.00046, 0.00045, 0.00047, 0.00046, 0.00045, 0.00045, 0.00049, 0.00045, 0.00049, 0.00045, 0.00045, 0.00046, 0.00045, 0.0005, 0.00045, 0.00046, 0.00044, 0.00045, 0.00049, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00049, 0.00046, 0.00048, 0.00047, 0.00045, 0.00045, 0.00046, 0.00048, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00046, 0.00048, 0.00048, 0.00048, 0.00048, 0.00045, 0.00045, 0.00048, 0.00047, 0.00045, 0.00048, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00047, 0.00047, 0.00046, 0.00046, 0.00047, 0.00047, 0.00044, 0.00045, 0.00045, 0.00048, 0.00048, 0.00048, 0.00045, 0.00045, 0.00046, 0.00045, 0.00048, 0.00048, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00049, 0.00045, 0.00045, 0.00045, 0.00045, 0.00048, 0.00045, 0.00046, 0.00049, 0.00046, 0.00046, 0.00044, 0.00048, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00048, 0.00047, 0.00049, 0.00045, 0.00045, 0.00053, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00049, 0.00045, 0.00044, 0.00048, 0.00045, 0.00045, 0.00045, 0.00045]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.13385, 0.00147, 0.00148, 0.00147, 0.00149, 0.00151, 0.00148, 0.00148, 0.00147, 0.00149, 0.00149, 0.00147, 0.00149, 0.00149, 0.00147, 0.00147, 0.00147, 0.00149, 0.00149, 0.00148, 0.00148, 0.00148, 0.0015, 0.0015, 0.00147, 0.00148, 0.00149, 0.00148, 0.00148, 0.00148, 0.00147, 0.00148, 0.00149, 0.00149, 0.00148, 0.00148, 0.00149, 0.00147, 0.00148, 0.00148, 0.00147, 0.00147, 0.00148, 0.00149, 0.00148, 0.00148, 0.00149, 0.00148, 0.00149, 0.00148, 0.00147, 0.00147, 0.00147, 0.00148, 0.00148, 0.00149, 0.00148, 0.00149, 0.00147, 0.00147, 0.00149, 0.00148, 0.00148, 0.00149, 0.0015, 0.00148, 0.00148, 0.00149, 0.00148, 0.00149, 0.00148, 0.00147, 0.00149, 0.00149, 0.00148, 0.00146, 0.00147, 0.00148, 0.00147, 0.00148, 0.00149, 0.00147, 0.00146, 0.00148, 0.00148, 0.00147, 0.00149, 0.00148, 0.00149, 0.0015, 0.00148, 0.00147, 0.00147, 0.00147, 0.00149, 0.00149, 0.00148, 0.00148, 0.00148, 0.00148, 0.00147, 0.00149, 0.00148, 0.00148, 0.00149, 0.00148, 0.00147, 0.00147, 0.00147, 0.00148, 0.00149, 0.00147, 0.00148, 0.00148, 0.00147, 0.00149, 0.00147, 0.00147, 0.00149, 0.00149, 0.00146, 0.00149, 0.00147, 0.00149, 0.00149, 0.00148, 0.00147, 0.00148, 0.00148, 0.00148, 0.00149, 0.00148, 0.00147, 0.00149, 0.00151, 0.00147, 0.00148, 0.00147, 0.00148, 0.00148, 0.00147, 0.00147, 0.0015, 0.00149, 0.00148, 0.00147, 0.00148, 0.00147, 0.00148, 0.00148, 0.00147, 0.0015, 0.00147, 0.00147, 0.00147, 0.00148, 0.0015, 0.00148, 0.00148, 0.00147, 0.00148, 0.00149, 0.00149, 0.00148, 0.00148, 0.00148, 0.00148, 0.00147, 0.00149, 0.00149, 0.00149, 0.00147, 0.00147, 0.00148, 0.00147, 0.00147, 0.00147, 0.00148, 0.00146, 0.00148, 0.00147, 0.00149, 0.00147, 0.00149, 0.00149, 0.00147, 0.00147, 0.00148, 0.00147, 0.00148, 0.00148, 0.00148, 0.00148, 0.00149, 0.00147, 0.00149, 0.00148, 0.00148, 0.00148, 0.00149, 0.0015, 0.00148, 0.00149, 0.00148, 0.00149, 0.00149, 0.00149, 0.00148, 0.00148, 0.00149, 0.00149, 0.0015, 0.00149, 0.00148, 0.00149, 0.00149, 0.00149, 0.00151, 0.00148, 0.0015, 0.00148, 0.00149, 0.00149, 0.00149, 0.00149, 0.00149, 0.00149, 0.0015, 0.0015, 0.0015, 0.00149, 0.0015, 0.00149, 0.00149, 0.00147, 0.00148, 0.00149, 0.0015, 0.0015, 0.00149, 0.00147, 0.00149, 0.0015, 0.00149, 0.00148, 0.00148, 0.00148, 0.00148, 0.00148, 0.0015, 0.0015, 0.0015, 0.00148, 0.00149, 0.00149, 0.00149, 0.00149, 0.00148, 0.0015, 0.00149, 0.00148, 0.00151, 0.00149, 0.00148, 0.00149, 0.00147, 0.00147, 0.00154, 0.00149, 0.00147, 0.00148, 0.0015, 0.00149, 0.00152, 0.00148, 0.00148, 0.00148, 0.00148, 0.00149, 0.00148, 0.00151, 0.00147, 0.00148, 0.00151, 0.0015, 0.00149, 0.00147, 0.00148, 0.00149, 0.00149, 0.00151, 0.00148, 0.00149, 0.00149, 0.00149, 0.00147, 0.00148, 0.00148, 0.00147, 0.00148, 0.00148, 0.00149, 0.00148, 0.00148, 0.00149, 0.00148, 0.00148, 0.00152, 0.00149, 0.0015, 0.00148, 0.00148, 0.00147, 0.00148, 0.00149, 0.00149, 0.00147, 0.00149, 0.00151, 0.00147, 0.00148, 0.00148, 0.00149, 0.00147, 0.0015, 0.00149, 0.00149, 0.00148, 0.00149, 0.00149, 0.00149, 0.00149, 0.00148, 0.00148, 0.00148, 0.00148, 0.00149, 0.00149, 0.00149, 0.00149, 0.00148, 0.00149, 0.00149, 0.00149, 0.00148, 0.0015, 0.00148, 0.00151, 0.00148, 0.00151, 0.00147, 0.00147, 0.00149, 0.00148, 0.00148, 0.00148, 0.00148, 0.00147, 0.00149, 0.00149, 0.00149, 0.00148, 0.00149, 0.0015, 0.00148, 0.00148, 0.00149, 0.00148, 0.00148, 0.00149, 0.00148, 0.00149, 0.0015, 0.00147, 0.00149, 0.00148, 0.00149, 0.00149, 0.00148, 0.00147, 0.00149, 0.0015, 0.0015, 0.00149, 0.00148, 0.00147, 0.00149, 0.00147, 0.0015, 0.00149, 0.00149, 0.00149, 0.0015, 0.00148, 0.00149, 0.00149, 0.0015, 0.00148, 0.00148, 0.00148]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00022, 0.00015, 0.00014, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00014, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00014, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00014, 0.00013, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00014, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00015, 0.00014, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00015, 0.00013, 0.00014, 0.00014, 0.00012, 0.00014, 0.00013, 0.00012, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00014, 0.00014, 0.00012, 0.00012, 0.00014, 0.00013, 0.00014, 0.00012, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00012, 0.00013, 0.00014, 0.00012, 0.00014, 0.00013, 0.00014, 0.00012, 0.00014, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00014, 0.00012, 0.00013, 0.00014, 0.00012, 0.00014, 0.00012, 0.00013, 0.00013, 0.00014, 0.00012, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00015, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00012, 0.00013, 0.00013, 0.00015, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00012, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00015, 0.00014, 0.00014, 0.00013, 0.00012, 0.00014, 0.00013, 0.00013, 0.00013, 0.00014, 0.00015, 0.00015, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00015, 0.00014, 0.00015, 0.00013, 0.00013, 0.00014, 0.00013, 0.00014, 0.00014, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00017, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.11156, 0.00067, 0.00064, 0.00065, 0.00062, 0.00063, 0.00062, 0.00063, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00067, 0.00062, 0.00063, 0.00063, 0.00063, 0.00063, 0.00062, 0.00062, 0.00061, 0.00062, 0.00062, 0.00062, 0.00064, 0.00064, 0.00064, 0.00063, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.00066, 0.00062, 0.00062, 0.00063, 0.00063, 0.00063, 0.00062, 0.00062, 0.00062, 0.00062, 0.00065, 0.00062, 0.00064, 0.00066, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.00065, 0.00065, 0.00064, 0.00063, 0.00062, 0.00064, 0.00063, 0.00062, 0.00067, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.00063, 0.00064, 0.00062, 0.00062, 0.00062, 0.00064, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.00062, 0.00064, 0.00063, 0.00064, 0.00063, 0.00066, 0.00062, 0.00062, 0.00062, 0.00061, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00061, 0.00062, 0.00071, 0.00046, 0.00069, 0.00062, 0.00068, 0.00062, 0.00062, 0.00045, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.0005, 0.00048, 0.00062, 0.00062, 0.00062, 0.00062, 0.00048, 0.00062, 0.00062, 0.00064, 0.00047, 0.00062, 0.00066, 0.00062, 0.00062, 0.00062, 0.00062, 0.00064, 0.00064, 0.00062, 0.00046, 0.00062, 0.00062, 0.00062, 0.00065, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00062, 0.00067, 0.00064, 0.00061, 0.00063, 0.00064, 0.00061, 0.00064, 0.00062, 0.00062, 0.00062, 0.00047, 0.00062, 0.00062, 0.00062, 0.00062, 0.00064, 0.00061, 0.00064, 0.00064, 0.00062, 0.00063, 0.00064, 0.00067, 0.00064, 0.00062, 0.00064, 0.00063, 0.00062, 0.00064, 0.00063, 0.00062, 0.00065, 0.00064, 0.00064, 0.00064, 0.00063, 0.00064, 0.00063, 0.00065, 0.00062, 0.00063, 0.00062, 0.00065, 0.00062, 0.00061, 0.00063, 0.00061, 0.00062, 0.00066, 0.00062, 0.00065, 0.00062, 0.00061, 0.00063, 0.00063, 0.00062, 0.00069, 0.00066, 0.00066, 0.00067, 0.00067, 0.00071, 0.00067, 0.00067, 0.00065, 0.00065, 0.00067, 0.00067, 0.00066, 0.00066, 0.00067, 0.00067, 0.00071, 0.00066, 0.00066, 0.00066, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00068, 0.00066, 0.00067, 0.00065, 0.00066, 0.00066, 0.00065, 0.00069, 0.00067, 0.00066, 0.00066, 0.00068, 0.00065, 0.00064, 0.00065, 0.00067, 0.00065, 0.00066, 0.00066, 0.00067, 0.00066, 0.00066, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00073, 0.00069, 0.00066, 0.00065, 0.00064, 0.00067, 0.00066, 0.00067, 0.00066, 0.00073, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00068, 0.00065, 0.00065, 0.00067, 0.00067, 0.00066, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00064, 0.00066, 0.00067, 0.00064, 0.00064, 0.00064, 0.00064, 0.00064, 0.00064, 0.00064, 0.00063, 0.00064, 0.00064, 0.00064, 0.00064, 0.00066, 0.00065, 0.00064, 0.00064, 0.00064, 0.00064, 0.00063, 0.00064, 0.00064, 0.00065, 0.00065, 0.00064, 0.00073, 0.00064, 0.00063, 0.00064, 0.00063, 0.00064, 0.00064, 0.00064, 0.00064, 0.00064, 0.00064, 0.00066, 0.00065, 0.00064, 0.00063, 0.00063, 0.00064, 0.00065, 0.00065, 0.00065, 0.00065, 0.00063, 0.00064, 0.00063, 0.00063, 0.00064, 0.00064, 0.00065, 0.00064, 0.00063, 0.00063, 0.00065, 0.00063, 0.00064, 0.00063, 0.00064, 0.00063, 0.00066, 0.00063, 0.00065, 0.00064, 0.00063, 0.00064, 0.00063, 0.00064, 0.00064, 0.00064, 0.00066, 0.00066, 0.00065, 0.00064, 0.00066, 0.00065, 0.00065, 0.00065, 0.00066, 0.00065, 0.00065, 0.00065, 0.00066, 0.00066, 0.00064, 0.00063, 0.00065, 0.00065, 0.00066, 0.00064, 0.00066, 0.00065, 0.00066, 0.00067, 0.00066, 0.00066, 0.00065, 0.00066, 0.00065, 0.00068, 0.00066, 0.00066, 0.00065, 0.00063, 0.00064, 0.00063, 0.00063, 0.00064]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00352, 0.00261, 0.00262, 0.00279, 0.00266, 0.00279, 0.00264, 0.00264, 0.00265, 0.00263, 0.00263, 0.00263, 0.00266, 0.00265, 0.00265, 0.00266, 0.00262, 0.00265, 0.00264, 0.00267, 0.00262, 0.00264, 0.00263, 0.00264, 0.00265, 0.00263, 0.00264, 0.00266, 0.00265, 0.00262, 0.00263, 0.00265, 0.00266, 0.00263, 0.00264, 0.00264, 0.00264, 0.00264, 0.00264, 0.00265, 0.00265, 0.00264, 0.00265, 0.00266, 0.00264, 0.00316, 0.00266, 0.00263, 0.00279, 0.0027, 0.00263, 0.00263, 0.00267, 0.00263, 0.00264, 0.00264, 0.00265, 0.00262, 0.00265, 0.00265, 0.00264, 0.00266, 0.00277, 0.00265, 0.00266, 0.00266, 0.00265, 0.00265, 0.00264, 0.00266, 0.00267, 0.00263, 0.00263, 0.00266, 0.00265, 0.00263, 0.00263, 0.00265, 0.00263, 0.00265, 0.00293, 0.00263, 0.00273, 0.00264, 0.00285, 0.00263, 0.00265, 0.00265, 0.00265, 0.00263, 0.00264, 0.00265, 0.00264, 0.00263, 0.00263, 0.00265, 0.00262, 0.00298, 0.00265, 0.0031, 0.00263, 0.00312, 0.00264, 0.00267, 0.00263, 0.00296, 0.00265, 0.00262, 0.00266, 0.00263, 0.00298, 0.00266, 0.00265, 0.00263, 0.00276, 0.00265, 0.00266, 0.00264, 0.00264, 0.00266, 0.00264, 0.00265, 0.00268, 0.00265, 0.00264, 0.00264, 0.00263, 0.00266, 0.00264, 0.00265, 0.00264, 0.00264, 0.00263, 0.00262, 0.00284, 0.00263, 0.00263, 0.00265, 0.00265, 0.00264, 0.00263, 0.00263, 0.00264, 0.00265, 0.00298, 0.00264, 0.00263, 0.00266, 0.00264, 0.00265, 0.00264, 0.00264, 0.00267, 0.00264, 0.00265, 0.00262, 0.00264, 0.00271, 0.00266, 0.00266, 0.00265, 0.00266, 0.00267, 0.00268, 0.00263, 0.00265, 0.00282, 0.00266, 0.0027, 0.00265, 0.00266, 0.00265, 0.00264, 0.00267, 0.00269, 0.00278, 0.00264, 0.00268, 0.00264, 0.00265, 0.00265, 0.00267, 0.00267, 0.00265, 0.00265, 0.00265, 0.00267, 0.00265, 0.00266, 0.00264, 0.00265, 0.00263, 0.00265, 0.00265, 0.00267, 0.00267, 0.00263, 0.00264, 0.00264, 0.00265, 0.00262, 0.00264, 0.00266, 0.00263, 0.00267, 0.00264, 0.00264, 0.00264, 0.00266, 0.00265, 0.00266, 0.00264, 0.00264, 0.00267, 0.00265, 0.00262, 0.00266, 0.00265, 0.00267, 0.00266, 0.00267, 0.00295, 0.00267, 0.00268, 0.00263, 0.00265, 0.00265, 0.00263, 0.00266, 0.00299, 0.00264, 0.00267, 0.00262, 0.00269, 0.00265, 0.00264, 0.00265, 0.00263, 0.00265, 0.00265, 0.00286, 0.00266, 0.00266, 0.00264, 0.00264, 0.00265, 0.00264, 0.00266, 0.00266, 0.00267, 0.00264, 0.00265, 0.00265, 0.00265, 0.00266, 0.00264, 0.00268, 0.00264, 0.00262, 0.00267, 0.00263, 0.00312, 0.00265, 0.00265, 0.00264, 0.00263, 0.00265, 0.00265, 0.00264, 0.00266, 0.00268, 0.00264, 0.00266, 0.00263, 0.00267, 0.00265, 0.00263, 0.00266, 0.0027, 0.00266, 0.00263, 0.00264, 0.00276, 0.00265, 0.00266, 0.00264, 0.00264, 0.00264, 0.00302, 0.00265, 0.00265, 0.00269, 0.00264, 0.00263, 0.00266, 0.00264, 0.00267, 0.00263, 0.00264, 0.00265, 0.00266, 0.00264, 0.00265, 0.00265, 0.00265, 0.00267, 0.00261, 0.00262, 0.00266, 0.00263, 0.00265, 0.00266, 0.00265, 0.00262, 0.00266, 0.00267, 0.00262, 0.00266, 0.00265, 0.00264, 0.00263, 0.00265, 0.00263, 0.00268, 0.00282, 0.00266, 0.00264, 0.00264, 0.00262, 0.00266, 0.00265, 0.00266, 0.00264, 0.00276, 0.00264, 0.00264, 0.00265, 0.00263, 0.00265, 0.00265, 0.00266, 0.00265, 0.00265, 0.00264, 0.00262, 0.00264, 0.00264, 0.00265, 0.00265, 0.00266, 0.00267, 0.00266, 0.00268, 0.00265, 0.00275, 0.00263, 0.00275, 0.00263, 0.00265, 0.00264, 0.00265, 0.00264, 0.00265, 0.00264, 0.00266, 0.00269, 0.00266, 0.00264, 0.00263, 0.00266, 0.00267, 0.00266, 0.00266, 0.00268, 0.00267, 0.00265, 0.00265, 0.00266, 0.00265, 0.00265, 0.00263, 0.00266, 0.00264, 0.00268, 0.00266, 0.00263, 0.00268, 0.00265, 0.00265, 0.00278, 0.0027, 0.00264, 0.00264, 0.00263, 0.00265, 0.00266, 0.00265, 0.00269, 0.00264, 0.00265]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0024, 0.00067, 0.00066, 0.00066, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00066, 0.00067, 0.00066, 0.00067, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00067, 0.00066, 0.00066, 0.00066, 0.00065, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00066, 0.00067, 0.00066, 0.00067, 0.00065, 0.00065, 0.00066, 0.0007, 0.00065, 0.00066, 0.00065, 0.00066, 0.00066, 0.00067, 0.00066, 0.00067, 0.00067, 0.00066, 0.00065, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00065, 0.00065, 0.00066, 0.00066, 0.00065, 0.00065, 0.00065, 0.00066, 0.00065, 0.00065, 0.00065, 0.00065, 0.00066, 0.00065, 0.00067, 0.00065, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00065, 0.00065, 0.00065, 0.00067, 0.00065, 0.00065, 0.00065, 0.00065, 0.00066, 0.00065, 0.00066, 0.00065, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00065, 0.00069, 0.00066, 0.00065, 0.00065, 0.00065, 0.00065, 0.00066, 0.00066, 0.00065, 0.00066, 0.00065, 0.00065, 0.00066, 0.00065, 0.00065, 0.00065, 0.00066, 0.00067, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00066, 0.00065, 0.00066, 0.00065, 0.00065, 0.00065, 0.00065, 0.00068, 0.00065, 0.00066, 0.00065, 0.00066, 0.00066, 0.00066, 0.00065, 0.00065, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00065, 0.00065, 0.00066, 0.00065, 0.00066, 0.00065, 0.00065, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00065, 0.00066, 0.00065, 0.00065, 0.00066, 0.00065, 0.00065, 0.00066, 0.00066, 0.00066, 0.00065, 0.00066, 0.00066, 0.00065, 0.00065, 0.00067, 0.00066, 0.00069, 0.00068, 0.00069, 0.00069, 0.00068, 0.0007, 0.00069, 0.00069, 0.00067, 0.00067, 0.00068, 0.00068, 0.00068, 0.00068, 0.00069, 0.00068, 0.00069, 0.00068, 0.00068, 0.00069, 0.00091, 0.00068, 0.00068, 0.00069, 0.00068, 0.00068, 0.00068, 0.00068, 0.00068, 0.00069, 0.00068, 0.00068, 0.00068, 0.00067, 0.00068, 0.00067, 0.00068, 0.00067, 0.00067, 0.00068, 0.00071, 0.00068, 0.00068, 0.00068, 0.00068, 0.00069, 0.00068, 0.00067, 0.00068, 0.00067, 0.0007, 0.00069, 0.00067, 0.00069, 0.00068, 0.00068, 0.00068, 0.00068, 0.00068, 0.00068, 0.00067, 0.00069, 0.00068, 0.00068, 0.00068, 0.00067, 0.00068, 0.00068, 0.00067, 0.00068, 0.00068, 0.00068, 0.00068, 0.00068, 0.00067, 0.00068, 0.00067, 0.00067, 0.00068, 0.00067, 0.00068, 0.00068, 0.00069, 0.00068, 0.00069, 0.00068, 0.00068, 0.00068, 0.00067, 0.00068, 0.00068, 0.00067, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00068, 0.00067, 0.00067, 0.00067, 0.00066, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00068, 0.00067, 0.00067, 0.00066, 0.00067, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00067, 0.00068, 0.00066, 0.00067, 0.00067, 0.00067, 0.00068, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00068, 0.00067, 0.00068, 0.00067, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00068, 0.00066, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00066, 0.00066, 0.00067, 0.00067, 0.00068, 0.00067, 0.00067, 0.00068, 0.00068, 0.00067, 0.00067, 0.00067, 0.00067, 0.00067, 0.00068, 0.00067, 0.00069, 0.00067, 0.00067, 0.00066, 0.00067, 0.00066, 0.00067, 0.00066]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0006, 0.00055, 0.00055, 0.00054, 0.00054, 0.00053, 0.00053, 0.00053, 0.00054, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00052, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00052, 0.00052, 0.00053, 0.00061, 0.00052, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00052, 0.00053, 0.00053, 0.00053, 0.00054, 0.00052, 0.00053, 0.00053, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00053, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00053, 0.00056, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00055, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00054, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00052, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00052, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00056, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00055, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00055, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00055, 0.00053, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00054, 0.00053, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00054, 0.00053, 0.00053, 0.00055, 0.00053, 0.00054, 0.00053, 0.00054, 0.00054, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00054, 0.00053, 0.00054, 0.00053, 0.00053, 0.00053, 0.00053, 0.00054, 0.0006]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.12049, 0.00501, 0.00496, 0.00513, 0.00494, 0.00512, 0.00493, 0.00495, 0.00494, 0.00491, 0.00493, 0.00491, 0.00494, 0.00492, 0.00498, 0.00492, 0.0049, 0.00495, 0.00492, 0.00497, 0.00492, 0.00491, 0.00492, 0.00492, 0.00492, 0.00491, 0.00496, 0.00498, 0.00494, 0.00491, 0.0049, 0.00492, 0.00494, 0.00492, 0.00491, 0.00497, 0.00492, 0.00491, 0.00492, 0.00493, 0.00493, 0.00491, 0.00492, 0.00494, 0.00492, 0.00556, 0.00493, 0.00491, 0.00512, 0.00512, 0.00492, 0.00493, 0.00494, 0.0049, 0.00494, 0.00495, 0.00496, 0.00491, 0.00491, 0.00496, 0.00492, 0.00493, 0.00512, 0.00493, 0.00493, 0.00494, 0.00491, 0.0049, 0.00491, 0.00496, 0.00492, 0.0049, 0.00489, 0.00495, 0.00491, 0.00488, 0.00493, 0.00491, 0.0049, 0.0049, 0.00526, 0.00491, 0.00503, 0.0049, 0.00519, 0.00488, 0.00492, 0.00491, 0.0049, 0.00491, 0.00489, 0.00491, 0.0049, 0.00487, 0.00489, 0.0049, 0.00489, 0.00539, 0.00473, 0.00548, 0.00489, 0.00551, 0.0049, 0.00493, 0.00471, 0.00529, 0.00491, 0.0049, 0.00491, 0.00489, 0.00522, 0.00479, 0.00492, 0.00492, 0.00503, 0.0049, 0.0048, 0.0049, 0.00492, 0.00494, 0.00475, 0.0049, 0.00498, 0.0049, 0.0049, 0.00489, 0.0049, 0.00536, 0.00494, 0.00492, 0.00474, 0.00491, 0.0049, 0.00491, 0.00516, 0.00489, 0.00491, 0.0049, 0.00492, 0.00493, 0.00506, 0.00489, 0.00489, 0.00491, 0.00534, 0.00497, 0.00488, 0.00496, 0.00493, 0.00489, 0.00494, 0.0049, 0.00493, 0.00492, 0.00478, 0.00489, 0.0049, 0.00501, 0.00493, 0.00496, 0.0049, 0.00496, 0.00496, 0.00496, 0.00492, 0.00494, 0.00516, 0.00496, 0.00497, 0.00495, 0.00494, 0.00494, 0.00493, 0.00496, 0.00494, 0.0051, 0.00495, 0.00495, 0.00493, 0.00492, 0.00495, 0.00493, 0.00498, 0.00491, 0.00494, 0.00492, 0.00496, 0.00491, 0.00491, 0.00493, 0.00492, 0.0049, 0.005, 0.00491, 0.00498, 0.00494, 0.00489, 0.00494, 0.00496, 0.00491, 0.00501, 0.00504, 0.00502, 0.00501, 0.00506, 0.00508, 0.00502, 0.00501, 0.00497, 0.00496, 0.005, 0.005, 0.00498, 0.00504, 0.00502, 0.00497, 0.00511, 0.00499, 0.00502, 0.00502, 0.00535, 0.00532, 0.00503, 0.00507, 0.005, 0.00501, 0.005, 0.00499, 0.00499, 0.00538, 0.00498, 0.00502, 0.00499, 0.00505, 0.00503, 0.00497, 0.00504, 0.00493, 0.00495, 0.00499, 0.00529, 0.00499, 0.00499, 0.00502, 0.00499, 0.00504, 0.00497, 0.00502, 0.005, 0.00501, 0.00503, 0.00504, 0.00496, 0.00502, 0.00502, 0.00501, 0.00503, 0.005, 0.00501, 0.00502, 0.00495, 0.00563, 0.00504, 0.005, 0.00496, 0.00494, 0.00501, 0.005, 0.00499, 0.0054, 0.00512, 0.00507, 0.00502, 0.005, 0.00501, 0.005, 0.00499, 0.00498, 0.00504, 0.00503, 0.00499, 0.00501, 0.00511, 0.00502, 0.00506, 0.00502, 0.00501, 0.00499, 0.00535, 0.00498, 0.00501, 0.00499, 0.00494, 0.00493, 0.00496, 0.00494, 0.00496, 0.00495, 0.00495, 0.00494, 0.00498, 0.00495, 0.00498, 0.00498, 0.00495, 0.005, 0.00492, 0.00493, 0.00494, 0.00492, 0.00498, 0.00494, 0.00496, 0.00495, 0.00497, 0.00506, 0.00494, 0.00497, 0.00498, 0.00495, 0.00494, 0.00495, 0.00497, 0.005, 0.00512, 0.00495, 0.00495, 0.00497, 0.00493, 0.00495, 0.00494, 0.00498, 0.00495, 0.00509, 0.005, 0.00498, 0.00493, 0.00494, 0.00496, 0.00495, 0.00497, 0.00495, 0.00495, 0.00496, 0.00491, 0.00494, 0.00498, 0.00494, 0.00494, 0.00495, 0.00496, 0.00495, 0.00501, 0.00495, 0.00508, 0.00493, 0.00505, 0.00493, 0.00494, 0.00495, 0.00495, 0.00496, 0.00501, 0.00497, 0.00499, 0.00499, 0.00499, 0.00495, 0.00494, 0.00498, 0.00498, 0.00498, 0.00497, 0.00499, 0.00499, 0.00497, 0.00494, 0.00495, 0.00497, 0.00497, 0.00496, 0.00496, 0.00496, 0.00501, 0.00501, 0.00497, 0.00503, 0.00498, 0.00498, 0.0051, 0.00507, 0.005, 0.00498, 0.00497, 0.00499, 0.00495, 0.00494, 0.00496, 0.00495, 0.00502]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.85966, 10.87073, 10.85528, 10.80344, 10.64111, 10.62649, 10.41586, 10.12808, 9.92567, 9.82477, 9.56932, 9.84031, 9.86916, 9.61422, 9.77599, 9.50086, 9.45226, 9.6411, 9.38013, 9.32634, 9.2385, 9.14186, 9.17287, 8.9927, 9.18814, 9.05768, 9.15476, 9.16458, 9.29864, 8.98678, 8.93067, 9.0473, 9.04611, 8.65648, 8.71651, 8.75511, 8.6848, 8.73632, 8.66102, 8.76482, 8.66202, 8.84911, 8.83074, 8.49813, 8.38745, 8.42847, 8.49038, 8.38199, 8.43014, 8.57752, 8.36366, 8.18998, 8.22416, 8.21877, 8.26315, 7.90938, 8.09005, 7.88773, 8.24, 8.22485, 7.99867, 7.95704, 7.91177, 7.73255, 7.73299, 7.63614, 7.50837, 7.90027, 7.69288, 7.44749, 7.73489, 7.76278, 7.53675, 7.29662, 7.44913, 7.33262, 7.46188, 7.22442, 7.63668, 7.27892, 7.3525, 7.21173, 7.21816, 7.422, 7.17639, 7.28501, 7.00259, 7.00597, 7.03995, 7.14192, 6.82608, 6.98941, 7.09192, 7.00491, 6.87719, 6.75925, 6.994, 7.05741, 6.70391, 6.57997, 6.72686, 6.74254, 6.73498, 6.73924, 6.65693, 6.40819, 6.63945, 6.61998, 6.44777, 6.63026, 6.7458, 6.60872, 6.72566, 6.6941, 6.62478, 6.5113, 6.60016, 6.40683, 6.66647, 6.25038, 6.25487, 6.30344, 6.39244, 6.35319, 6.45279, 6.29501, 6.34432, 6.24122, 6.20479, 6.40226, 6.3298, 6.33253, 6.17365, 6.1703, 6.25122, 6.39707, 6.21313, 6.16095, 6.19193, 6.12904, 6.07716, 6.08434, 6.27156, 6.42116, 6.27092, 6.31502, 6.1099, 6.19051, 6.01202, 6.04186, 5.96572, 6.2566, 6.1994, 5.97238, 5.79066, 6.13517, 5.8567, 6.11381, 5.79621, 6.16806, 6.15725, 6.09481, 5.94172, 6.12313, 5.95406, 6.20205, 5.90266, 5.80426, 5.78673, 5.69691, 6.02057, 6.00205, 6.07073, 5.89354, 6.04415, 5.97229, 5.99763, 5.99201, 5.9504, 5.83989, 5.95152, 5.61741, 5.70128, 5.88995, 5.84414, 5.86222, 5.76021, 5.83835, 5.72362, 5.56328, 5.72206, 5.62699, 5.83296, 5.60473, 5.71241, 5.71399, 5.89863, 5.64481, 5.85045, 5.74116, 5.86786, 5.33069, 5.89739, 5.87147, 5.85621, 5.41402, 5.40885, 5.6244, 5.5909, 5.48288, 5.57328, 5.66993, 5.47325, 5.74532, 5.50733, 5.58951, 5.62335, 5.61873, 5.50712, 5.61686, 5.67259, 5.68325, 5.58652, 5.65724, 5.37154, 5.68206, 5.62545, 5.42293, 5.5898, 5.63487, 5.55215, 5.34318, 5.53918, 5.48775, 5.48384, 5.38046, 5.5524, 5.6054, 5.39011, 5.52269, 5.48564, 5.33339, 5.50751, 5.41235, 5.44463, 5.32284, 5.07354, 5.47834, 5.57158, 5.71691, 5.41899, 5.60533, 5.64283, 5.2342, 5.27417, 5.39872, 5.39954, 5.33267, 5.50546, 5.18598, 5.3031, 5.25146, 5.37886, 5.25856, 5.45542, 5.53656, 5.3141, 5.4389, 5.34171, 5.07715, 5.31356, 5.26151, 5.30932, 5.1132, 5.27888, 5.26913, 5.47802, 5.16411, 5.27179, 5.21046, 5.36047, 4.98558, 4.92161, 5.33001, 5.39104, 5.23106, 5.32226, 5.1108, 5.16307, 5.26011, 5.06878, 5.26621, 5.0712, 5.34447, 5.24947, 5.15197, 5.24511, 5.04213, 5.3173, 5.05677, 5.03031, 5.14366, 5.11315, 5.27152, 5.15384, 5.27818, 5.09471, 5.09718, 5.25022, 5.32221, 5.25368, 5.19177, 5.14141, 5.29041, 4.95105, 5.2074, 5.08987, 5.30215, 5.17471, 5.18799, 5.1137, 4.98327, 4.99184, 5.2222, 5.31185, 5.09737, 5.05507, 4.91447, 5.12386, 5.11467, 4.92535, 5.33586, 5.02667, 5.10506, 5.16491, 5.00221, 5.06296, 5.06915, 4.9949, 5.07922, 5.16029, 4.97927, 5.18201, 4.92792, 4.92204, 5.06399, 4.99471, 4.90735, 4.77765, 4.94535, 5.11795, 5.01969, 5.02225, 5.33057, 4.96058, 4.9931, 5.0457, 4.81181, 4.74328, 4.99687, 5.0383, 4.87423, 4.95276, 5.04325, 5.02264, 4.81956, 4.89599, 4.90754, 4.8294, 4.74438, 5.01179, 4.75262, 5.2095, 4.78557, 4.99344, 4.73813, 4.78739, 4.82401, 4.64885, 4.65631, 4.84474, 4.80822, 4.80327, 4.92878, 4.88473, 4.93264, 4.7706, 4.88531, 4.73767, 4.91524, 4.95719, 4.87814, 4.70608, 4.7878, 4.89822, 4.71172, 4.87123, 4.69258, 4.69633, 4.64631]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.85966, 10.87073, 10.85528, 10.80344, 10.64111, 10.62649, 10.41586, 10.12808, 9.92567, 9.82477, 9.56932, 9.84031, 9.86916, 9.61422, 9.77599, 9.50086, 9.45226, 9.6411, 9.38013, 9.32634, 9.2385, 9.14186, 9.17287, 8.9927, 9.18814, 9.05768, 9.15476, 9.16458, 9.29864, 8.98678, 8.93067, 9.0473, 9.04611, 8.65648, 8.71651, 8.75511, 8.6848, 8.73632, 8.66102, 8.76482, 8.66202, 8.84911, 8.83074, 8.49813, 8.38745, 8.42847, 8.49038, 8.38199, 8.43014, 8.57752, 8.36366, 8.18998, 8.22416, 8.21877, 8.26315, 7.90938, 8.09005, 7.88773, 8.24, 8.22485, 7.99867, 7.95704, 7.91177, 7.73255, 7.73299, 7.63614, 7.50837, 7.90027, 7.69288, 7.44749, 7.73489, 7.76278, 7.53675, 7.29662, 7.44913, 7.33262, 7.46188, 7.22442, 7.63668, 7.27892, 7.3525, 7.21173, 7.21816, 7.422, 7.17639, 7.28501, 7.00259, 7.00597, 7.03995, 7.14192, 6.82608, 6.98941, 7.09192, 7.00491, 6.87719, 6.75925, 6.994, 7.05741, 6.70391, 6.57997, 6.72686, 6.74254, 6.73498, 6.73924, 6.65693, 6.40819, 6.63945, 6.61998, 6.44777, 6.63026, 6.7458, 6.60872, 6.72566, 6.6941, 6.62478, 6.5113, 6.60016, 6.40683, 6.66647, 6.25038, 6.25487, 6.30344, 6.39244, 6.35319, 6.45279, 6.29501, 6.34432, 6.24122, 6.20479, 6.40226, 6.3298, 6.33253, 6.17365, 6.1703, 6.25122, 6.39707, 6.21313, 6.16095, 6.19193, 6.12904, 6.07716, 6.08434, 6.27156, 6.42116, 6.27092, 6.31502, 6.1099, 6.19051, 6.01202, 6.04186, 5.96572, 6.2566, 6.1994, 5.97238, 5.79066, 6.13517, 5.8567, 6.11381, 5.79621, 6.16806, 6.15725, 6.09481, 5.94172, 6.12313, 5.95406, 6.20205, 5.90266, 5.80426, 5.78673, 5.69691, 6.02057, 6.00205, 6.07073, 5.89354, 6.04415, 5.97229, 5.99763, 5.99201, 5.9504, 5.83989, 5.95152, 5.61741, 5.70128, 5.88995, 5.84414, 5.86222, 5.76021, 5.83835, 5.72362, 5.56328, 5.72206, 5.62699, 5.83296, 5.60473, 5.71241, 5.71399, 5.89863, 5.64481, 5.85045, 5.74116, 5.86786, 5.33069, 5.89739, 5.87147, 5.85621, 5.41402, 5.40885, 5.6244, 5.5909, 5.48288, 5.57328, 5.66993, 5.47325, 5.74532, 5.50733, 5.58951, 5.62335, 5.61873, 5.50712, 5.61686, 5.67259, 5.68325, 5.58652, 5.65724, 5.37154, 5.68206, 5.62545, 5.42293, 5.5898, 5.63487, 5.55215, 5.34318, 5.53918, 5.48775, 5.48384, 5.38046, 5.5524, 5.6054, 5.39011, 5.52269, 5.48564, 5.33339, 5.50751, 5.41235, 5.44463, 5.32284, 5.07354, 5.47834, 5.57158, 5.71691, 5.41899, 5.60533, 5.64283, 5.2342, 5.27417, 5.39872, 5.39954, 5.33267, 5.50546, 5.18598, 5.3031, 5.25146, 5.37886, 5.25856, 5.45542, 5.53656, 5.3141, 5.4389, 5.34171, 5.07715, 5.31356, 5.26151, 5.30932, 5.1132, 5.27888, 5.26913, 5.47802, 5.16411, 5.27179, 5.21046, 5.36047, 4.98558, 4.92161, 5.33001, 5.39104, 5.23106, 5.32226, 5.1108, 5.16307, 5.26011, 5.06878, 5.26621, 5.0712, 5.34447, 5.24947, 5.15197, 5.24511, 5.04213, 5.3173, 5.05677, 5.03031, 5.14366, 5.11315, 5.27152, 5.15384, 5.27818, 5.09471, 5.09718, 5.25022, 5.32221, 5.25368, 5.19177, 5.14141, 5.29041, 4.95105, 5.2074, 5.08987, 5.30215, 5.17471, 5.18799, 5.1137, 4.98327, 4.99184, 5.2222, 5.31185, 5.09737, 5.05507, 4.91447, 5.12386, 5.11467, 4.92535, 5.33586, 5.02667, 5.10506, 5.16491, 5.00221, 5.06296, 5.06915, 4.9949, 5.07922, 5.16029, 4.97927, 5.18201, 4.92792, 4.92204, 5.06399, 4.99471, 4.90735, 4.77765, 4.94535, 5.11795, 5.01969, 5.02225, 5.33057, 4.96058, 4.9931, 5.0457, 4.81181, 4.74328, 4.99687, 5.0383, 4.87423, 4.95276, 5.04325, 5.02264, 4.81956, 4.89599, 4.90754, 4.8294, 4.74438, 5.01179, 4.75262, 5.2095, 4.78557, 4.99344, 4.73813, 4.78739, 4.82401, 4.64885, 4.65631, 4.84474, 4.80822, 4.80327, 4.92878, 4.88473, 4.93264, 4.7706, 4.88531, 4.73767, 4.91524, 4.95719, 4.87814, 4.70608, 4.7878, 4.89822, 4.71172, 4.87123, 4.69258, 4.69633, 4.64631]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.56517, 13.52183, 13.82389, 12.68199, 12.11513, 9.42628, 6.78009, 6.96682, 6.03524, 4.63457, 4.1513, 2.87067, 2.35463, 2.3279, 2.02459, 2.22441, 2.16108, 1.87618, 2.21105, 2.06296, 2.12729, 2.152, 2.00687, 2.2248, 1.98285, 2.1147, 1.92124, 1.92395, 1.94527, 2.15653, 2.0865, 1.94545, 1.87214, 2.15774, 2.14492, 2.10813, 1.99702, 1.84398, 1.93326, 1.73194, 2.15655, 1.83365, 1.74796, 1.87637, 1.87935, 1.82812, 1.70882, 1.75031, 1.75541, 1.56033, 1.72362, 1.80715, 1.77318, 1.81611, 1.66844, 1.80559, 1.7625, 1.84598, 1.62632, 1.48661, 1.64786, 1.45473, 1.77763, 1.80854, 1.64942, 1.65627, 1.70353, 1.60171, 1.44031, 1.72339, 1.43433, 1.37767, 1.68581, 1.37671, 1.40648, 1.61691, 1.50881, 1.38382, 1.44532, 1.27357, 1.36667, 1.33118, 1.30365, 1.39513, 1.39043, 1.4631, 1.55974, 1.45774, 1.22995, 1.11972, 1.09726, 1.20059, 1.10224, 1.31175, 1.01034, 1.30362, 1.38885, 1.05046, 0.94787, 1.76252, 1.11012, 1.2148, 1.71468, 1.62278, 0.95552, 1.16789, 1.17655, 1.03922, 1.21282, 1.1032, 0.98669, 0.95678, 1.1193, 1.05737, 1.01498, 1.16799, 0.97578, 1.42941, 1.13594, 1.05985, 0.9398, 1.10182, 1.02064, 1.3517, 1.44708, 2.04415, 1.69036, 1.40806, 1.38738, 1.3424, 0.99552, 1.67778, 1.38915, 1.16703, 1.21285, 1.27027, 1.08112, 1.56529, 1.11243, 1.55047, 1.88478, 1.49661, 1.24747, 1.30858, 1.0413, 1.79193, 1.1894, 1.10832, 1.14553, 1.37473, 1.12916, 1.19043, 1.55147, 1.14787, 0.9831, 1.97748, 1.30968, 1.75548, 1.42903, 1.47772, 1.63806, 1.08487, 1.3989, 1.02365, 1.24838, 1.43469, 1.42662, 1.30881, 1.20964, 1.49347, 1.21919, 1.05332, 1.18399, 1.38555, 1.13727, 1.36432, 1.2528, 1.17022, 1.32348, 1.07935, 1.19539, 1.48684, 1.19029, 1.2198, 1.81559, 1.52452, 1.79334, 1.66013, 1.20616, 1.67532, 1.19437, 1.28, 1.33364, 1.69679, 1.53842, 1.37202, 1.34387, 1.37081, 1.28649, 1.5618, 1.03326, 1.39685, 1.27238, 1.20598, 1.32922, 1.41054, 1.32813, 1.46075, 1.18533, 1.18314, 1.37783, 1.39264, 1.2322, 1.35301, 1.51994, 1.29479, 1.54145, 1.57876, 1.23038, 1.67935, 1.59903, 1.7688, 1.38891, 1.39714, 1.41056, 1.56263, 1.84649, 1.31226, 2.25632, 1.5966, 1.20159, 1.49708, 1.73963, 1.47932, 1.74434, 1.84578, 1.28148, 1.58712, 1.57826, 1.14575, 1.37743, 1.14726, 1.36495, 1.54092, 1.1998, 1.83908, 1.60608, 1.22735, 1.39352, 1.48052, 1.44922, 1.5986, 1.86828, 1.2133, 1.28534, 1.44591, 1.40707, 1.6217, 1.68123, 1.16996, 1.40545, 1.79994, 1.32408, 1.35454, 1.82216, 1.50619, 1.25331, 1.36593, 1.33067, 1.20379, 1.1715, 1.34612, 1.23828, 1.2249, 1.23199, 1.50931, 1.24187, 1.31666, 1.33544, 1.15247, 1.35164, 1.31814, 1.51121, 1.22179, 1.26518, 1.48248, 1.47105, 2.08081, 1.48841, 1.53234, 1.46321, 1.4755, 1.16048, 1.44268, 1.5642, 1.52523, 1.38495, 1.80119, 1.63483, 1.41261, 1.60553, 1.28802, 1.15347, 1.54912, 1.53753, 1.36296, 1.66631, 1.63888, 1.24348, 1.42956, 1.32686, 1.487, 1.7063, 1.383, 1.67566, 1.4665, 1.41433, 1.44807, 1.36307, 1.13744, 1.63129, 1.56395, 1.59787, 1.49857, 1.45091, 1.60777, 1.36633, 1.34096, 1.63579, 1.34741, 1.48819, 1.66258, 1.532, 1.46235, 1.36272, 1.36735, 1.33239, 1.3176, 1.2966, 1.56971, 1.31551, 1.50053, 1.27598, 1.29926, 1.5045, 1.39074, 1.41138, 1.40198, 1.46432, 1.38696, 1.52639, 1.55526, 1.4432, 1.27923, 1.48503, 1.17404, 1.20825, 1.60545, 1.81024, 1.35059, 1.28697, 1.50174, 1.46699, 1.33784, 1.08159, 1.61115, 1.46019, 1.37898, 1.35614, 1.65157, 1.46597, 1.60688, 1.72399, 1.30124, 1.44364, 1.32297, 1.13212, 1.45342, 1.38164, 1.21948, 1.26404, 1.33477, 1.30704, 1.51357, 1.26848, 1.55252, 1.33368, 1.41811, 1.47778, 1.31706, 1.20105, 1.48475, 1.28543, 1.46568, 1.42638, 1.25259, 1.60254, 1.36812, 1.3586, 1.15672]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.56517, 13.52183, 13.82389, 12.68199, 12.11513, 9.42628, 6.78009, 6.96682, 6.03524, 4.63457, 4.1513, 2.87067, 2.35463, 2.3279, 2.02459, 2.22441, 2.16108, 1.87618, 2.21105, 2.06296, 2.12729, 2.152, 2.00687, 2.2248, 1.98285, 2.1147, 1.92124, 1.92395, 1.94527, 2.15653, 2.0865, 1.94545, 1.87214, 2.15774, 2.14492, 2.10813, 1.99702, 1.84398, 1.93326, 1.73194, 2.15655, 1.83365, 1.74796, 1.87637, 1.87935, 1.82812, 1.70882, 1.75031, 1.75541, 1.56033, 1.72362, 1.80715, 1.77318, 1.81611, 1.66844, 1.80559, 1.7625, 1.84598, 1.62632, 1.48661, 1.64786, 1.45473, 1.77763, 1.80854, 1.64942, 1.65627, 1.70353, 1.60171, 1.44031, 1.72339, 1.43433, 1.37767, 1.68581, 1.37671, 1.40648, 1.61691, 1.50881, 1.38382, 1.44532, 1.27357, 1.36667, 1.33118, 1.30365, 1.39513, 1.39043, 1.4631, 1.55974, 1.45774, 1.22995, 1.11972, 1.09726, 1.20059, 1.10224, 1.31175, 1.01034, 1.30362, 1.38885, 1.05046, 0.94787, 1.76252, 1.11012, 1.2148, 1.71468, 1.62278, 0.95552, 1.16789, 1.17655, 1.03922, 1.21282, 1.1032, 0.98669, 0.95678, 1.1193, 1.05737, 1.01498, 1.16799, 0.97578, 1.42941, 1.13594, 1.05985, 0.9398, 1.10182, 1.02064, 1.3517, 1.44708, 2.04415, 1.69036, 1.40806, 1.38738, 1.3424, 0.99552, 1.67778, 1.38915, 1.16703, 1.21285, 1.27027, 1.08112, 1.56529, 1.11243, 1.55047, 1.88478, 1.49661, 1.24747, 1.30858, 1.0413, 1.79193, 1.1894, 1.10832, 1.14553, 1.37473, 1.12916, 1.19043, 1.55147, 1.14787, 0.9831, 1.97748, 1.30968, 1.75548, 1.42903, 1.47772, 1.63806, 1.08487, 1.3989, 1.02365, 1.24838, 1.43469, 1.42662, 1.30881, 1.20964, 1.49347, 1.21919, 1.05332, 1.18399, 1.38555, 1.13727, 1.36432, 1.2528, 1.17022, 1.32348, 1.07935, 1.19539, 1.48684, 1.19029, 1.2198, 1.81559, 1.52452, 1.79334, 1.66013, 1.20616, 1.67532, 1.19437, 1.28, 1.33364, 1.69679, 1.53842, 1.37202, 1.34387, 1.37081, 1.28649, 1.5618, 1.03326, 1.39685, 1.27238, 1.20598, 1.32922, 1.41054, 1.32813, 1.46075, 1.18533, 1.18314, 1.37783, 1.39264, 1.2322, 1.35301, 1.51994, 1.29479, 1.54145, 1.57876, 1.23038, 1.67935, 1.59903, 1.7688, 1.38891, 1.39714, 1.41056, 1.56263, 1.84649, 1.31226, 2.25632, 1.5966, 1.20159, 1.49708, 1.73963, 1.47932, 1.74434, 1.84578, 1.28148, 1.58712, 1.57826, 1.14575, 1.37743, 1.14726, 1.36495, 1.54092, 1.1998, 1.83908, 1.60608, 1.22735, 1.39352, 1.48052, 1.44922, 1.5986, 1.86828, 1.2133, 1.28534, 1.44591, 1.40707, 1.6217, 1.68123, 1.16996, 1.40545, 1.79994, 1.32408, 1.35454, 1.82216, 1.50619, 1.25331, 1.36593, 1.33067, 1.20379, 1.1715, 1.34612, 1.23828, 1.2249, 1.23199, 1.50931, 1.24187, 1.31666, 1.33544, 1.15247, 1.35164, 1.31814, 1.51121, 1.22179, 1.26518, 1.48248, 1.47105, 2.08081, 1.48841, 1.53234, 1.46321, 1.4755, 1.16048, 1.44268, 1.5642, 1.52523, 1.38495, 1.80119, 1.63483, 1.41261, 1.60553, 1.28802, 1.15347, 1.54912, 1.53753, 1.36296, 1.66631, 1.63888, 1.24348, 1.42956, 1.32686, 1.487, 1.7063, 1.383, 1.67566, 1.4665, 1.41433, 1.44807, 1.36307, 1.13744, 1.63129, 1.56395, 1.59787, 1.49857, 1.45091, 1.60777, 1.36633, 1.34096, 1.63579, 1.34741, 1.48819, 1.66258, 1.532, 1.46235, 1.36272, 1.36735, 1.33239, 1.3176, 1.2966, 1.56971, 1.31551, 1.50053, 1.27598, 1.29926, 1.5045, 1.39074, 1.41138, 1.40198, 1.46432, 1.38696, 1.52639, 1.55526, 1.4432, 1.27923, 1.48503, 1.17404, 1.20825, 1.60545, 1.81024, 1.35059, 1.28697, 1.50174, 1.46699, 1.33784, 1.08159, 1.61115, 1.46019, 1.37898, 1.35614, 1.65157, 1.46597, 1.60688, 1.72399, 1.30124, 1.44364, 1.32297, 1.13212, 1.45342, 1.38164, 1.21948, 1.26404, 1.33477, 1.30704, 1.51357, 1.26848, 1.55252, 1.33368, 1.41811, 1.47778, 1.31706, 1.20105, 1.48475, 1.28543, 1.46568, 1.42638, 1.25259, 1.60254, 1.36812, 1.3586, 1.15672]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [78.0, 71.0, 69.0, 77.0, 83.0, 93.0, 106.0, 92.0, 92.0, 132.0, 100.0, 151.0, 124.0, 174.0, 156.0, 150.0, 169.0, 195.0, 167.0, 147.0, 152.0, 152.0, 200.0, 189.0, 169.0, 153.0, 197.0, 164.0, 147.0, 172.0, 144.0, 157.0, 169.0, 165.0, 146.0, 179.0, 172.0, 212.0, 186.0, 196.0, 171.0, 138.0, 152.0, 197.0, 156.0, 167.0, 212.0, 178.0, 187.0, 180.0, 190.0, 159.0, 176.0, 163.0, 179.0, 191.0, 150.0, 150.0, 227.0, 225.0, 197.0, 184.0, 184.0, 199.0, 214.0, 235.0, 186.0, 197.0, 214.0, 222.0, 193.0, 241.0, 159.0, 264.0, 193.0, 187.0, 201.0, 208.0, 227.0, 223.0, 225.0, 212.0, 231.0, 219.0, 202.0, 196.0, 178.0, 182.0, 185.0, 210.0, 201.0, 198.0, 213.0, 214.0, 205.0, 161.0, 183.0, 193.0, 198.0, 178.0, 190.0, 166.0, 137.0, 154.0, 183.0, 150.0, 165.0, 166.0, 127.0, 174.0, 160.0, 171.0, 188.0, 172.0, 159.0, 152.0, 151.0, 127.0, 137.0, 145.0, 172.0, 135.0, 151.0, 158.0, 141.0, 113.0, 114.0, 93.0, 113.0, 128.0, 148.0, 125.0, 114.0, 127.0, 121.0, 117.0, 146.0, 116.0, 148.0, 137.0, 108.0, 114.0, 129.0, 141.0, 130.0, 107.0, 113.0, 126.0, 130.0, 102.0, 127.0, 110.0, 108.0, 109.0, 112.0, 65.0, 98.0, 84.0, 105.0, 108.0, 95.0, 135.0, 103.0, 123.0, 101.0, 102.0, 101.0, 117.0, 109.0, 106.0, 123.0, 114.0, 102.0, 88.0, 131.0, 104.0, 116.0, 108.0, 142.0, 118.0, 121.0, 115.0, 118.0, 115.0, 106.0, 119.0, 105.0, 84.0, 106.0, 91.0, 120.0, 114.0, 140.0, 96.0, 85.0, 100.0, 114.0, 103.0, 153.0, 88.0, 120.0, 96.0, 122.0, 111.0, 89.0, 107.0, 111.0, 97.0, 128.0, 103.0, 123.0, 90.0, 94.0, 82.0, 100.0, 109.0, 112.0, 104.0, 119.0, 90.0, 77.0, 114.0, 82.0, 103.0, 104.0, 104.0, 97.0, 127.0, 67.0, 99.0, 126.0, 90.0, 84.0, 109.0, 94.0, 97.0, 107.0, 113.0, 127.0, 100.0, 115.0, 102.0, 96.0, 116.0, 125.0, 102.0, 91.0, 126.0, 114.0, 101.0, 113.0, 110.0, 96.0, 126.0, 121.0, 99.0, 104.0, 108.0, 86.0, 143.0, 120.0, 83.0, 115.0, 92.0, 73.0, 113.0, 117.0, 111.0, 93.0, 106.0, 131.0, 93.0, 121.0, 109.0, 108.0, 115.0, 117.0, 116.0, 105.0, 110.0, 103.0, 112.0, 85.0, 118.0, 126.0, 119.0, 120.0, 104.0, 112.0, 111.0, 108.0, 107.0, 126.0, 123.0, 100.0, 81.0, 101.0, 106.0, 93.0, 109.0, 104.0, 131.0, 134.0, 98.0, 105.0, 129.0, 83.0, 87.0, 128.0, 116.0, 114.0, 111.0, 94.0, 114.0, 91.0, 97.0, 93.0, 116.0, 135.0, 122.0, 111.0, 126.0, 107.0, 107.0, 101.0, 82.0, 120.0, 142.0, 124.0, 120.0, 124.0, 122.0, 97.0, 96.0, 107.0, 102.0, 123.0, 115.0, 126.0, 116.0, 122.0, 115.0, 107.0, 111.0, 95.0, 93.0, 113.0, 117.0, 101.0, 110.0, 126.0, 113.0, 112.0, 127.0, 138.0, 118.0, 133.0, 94.0, 105.0, 119.0, 121.0, 122.0, 102.0, 98.0, 119.0, 103.0, 108.0, 134.0, 116.0, 107.0, 105.0, 99.0, 99.0, 117.0, 106.0, 133.0, 108.0, 110.0, 99.0, 140.0, 107.0, 104.0, 114.0, 112.0, 117.0, 106.0, 105.0, 92.0, 111.0, 99.0, 124.0, 101.0, 102.0, 144.0, 129.0, 122.0, 110.0, 116.0, 123.0, 136.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [78.0, 71.0, 69.0, 77.0, 83.0, 93.0, 106.0, 92.0, 92.0, 132.0, 100.0, 151.0, 124.0, 174.0, 156.0, 150.0, 169.0, 195.0, 167.0, 147.0, 152.0, 152.0, 200.0, 189.0, 169.0, 153.0, 197.0, 164.0, 147.0, 172.0, 144.0, 157.0, 169.0, 165.0, 146.0, 179.0, 172.0, 212.0, 186.0, 196.0, 171.0, 138.0, 152.0, 197.0, 156.0, 167.0, 212.0, 178.0, 187.0, 180.0, 190.0, 159.0, 176.0, 163.0, 179.0, 191.0, 150.0, 150.0, 227.0, 225.0, 197.0, 184.0, 184.0, 199.0, 214.0, 235.0, 186.0, 197.0, 214.0, 222.0, 193.0, 241.0, 159.0, 264.0, 193.0, 187.0, 201.0, 208.0, 227.0, 223.0, 225.0, 212.0, 231.0, 219.0, 202.0, 196.0, 178.0, 182.0, 185.0, 210.0, 201.0, 198.0, 213.0, 214.0, 205.0, 161.0, 183.0, 193.0, 198.0, 178.0, 190.0, 166.0, 137.0, 154.0, 183.0, 150.0, 165.0, 166.0, 127.0, 174.0, 160.0, 171.0, 188.0, 172.0, 159.0, 152.0, 151.0, 127.0, 137.0, 145.0, 172.0, 135.0, 151.0, 158.0, 141.0, 113.0, 114.0, 93.0, 113.0, 128.0, 148.0, 125.0, 114.0, 127.0, 121.0, 117.0, 146.0, 116.0, 148.0, 137.0, 108.0, 114.0, 129.0, 141.0, 130.0, 107.0, 113.0, 126.0, 130.0, 102.0, 127.0, 110.0, 108.0, 109.0, 112.0, 65.0, 98.0, 84.0, 105.0, 108.0, 95.0, 135.0, 103.0, 123.0, 101.0, 102.0, 101.0, 117.0, 109.0, 106.0, 123.0, 114.0, 102.0, 88.0, 131.0, 104.0, 116.0, 108.0, 142.0, 118.0, 121.0, 115.0, 118.0, 115.0, 106.0, 119.0, 105.0, 84.0, 106.0, 91.0, 120.0, 114.0, 140.0, 96.0, 85.0, 100.0, 114.0, 103.0, 153.0, 88.0, 120.0, 96.0, 122.0, 111.0, 89.0, 107.0, 111.0, 97.0, 128.0, 103.0, 123.0, 90.0, 94.0, 82.0, 100.0, 109.0, 112.0, 104.0, 119.0, 90.0, 77.0, 114.0, 82.0, 103.0, 104.0, 104.0, 97.0, 127.0, 67.0, 99.0, 126.0, 90.0, 84.0, 109.0, 94.0, 97.0, 107.0, 113.0, 127.0, 100.0, 115.0, 102.0, 96.0, 116.0, 125.0, 102.0, 91.0, 126.0, 114.0, 101.0, 113.0, 110.0, 96.0, 126.0, 121.0, 99.0, 104.0, 108.0, 86.0, 143.0, 120.0, 83.0, 115.0, 92.0, 73.0, 113.0, 117.0, 111.0, 93.0, 106.0, 131.0, 93.0, 121.0, 109.0, 108.0, 115.0, 117.0, 116.0, 105.0, 110.0, 103.0, 112.0, 85.0, 118.0, 126.0, 119.0, 120.0, 104.0, 112.0, 111.0, 108.0, 107.0, 126.0, 123.0, 100.0, 81.0, 101.0, 106.0, 93.0, 109.0, 104.0, 131.0, 134.0, 98.0, 105.0, 129.0, 83.0, 87.0, 128.0, 116.0, 114.0, 111.0, 94.0, 114.0, 91.0, 97.0, 93.0, 116.0, 135.0, 122.0, 111.0, 126.0, 107.0, 107.0, 101.0, 82.0, 120.0, 142.0, 124.0, 120.0, 124.0, 122.0, 97.0, 96.0, 107.0, 102.0, 123.0, 115.0, 126.0, 116.0, 122.0, 115.0, 107.0, 111.0, 95.0, 93.0, 113.0, 117.0, 101.0, 110.0, 126.0, 113.0, 112.0, 127.0, 138.0, 118.0, 133.0, 94.0, 105.0, 119.0, 121.0, 122.0, 102.0, 98.0, 119.0, 103.0, 108.0, 134.0, 116.0, 107.0, 105.0, 99.0, 99.0, 117.0, 106.0, 133.0, 108.0, 110.0, 99.0, 140.0, 107.0, 104.0, 114.0, 112.0, 117.0, 106.0, 105.0, 92.0, 111.0, 99.0, 124.0, 101.0, 102.0, 144.0, 129.0, 122.0, 110.0, 116.0, 123.0, 136.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.94354, 179.94354, 179.94354, 179.94353, 179.94351, 179.94351, 179.9435, 179.94337, 179.94319, 179.94301, 179.94168, 179.94092, 179.94034, 179.9382, 179.93718, 179.93637, 179.93611, 179.93633, 179.93683, 179.93695, 179.93684, 179.93649, 179.9361, 179.93663, 179.93771, 179.93913, 179.94032, 179.94113, 179.94214, 179.94365, 179.94586, 179.94824, 179.95052, 179.95296, 179.95572, 179.95921, 179.96291, 179.96681, 179.97093, 179.97545, 179.98062, 179.98616, 179.99197, 179.99846, 180.00552, 180.01314, 180.02119, 180.03004, 180.0396, 180.05011, 180.06131, 180.07315, 180.08542, 180.0985, 180.11215, 180.12645, 180.14087, 180.15598, 180.17198, 180.18895, 180.20711, 180.22621, 180.24666, 180.26831, 180.28981, 180.31268, 180.33565, 180.35945, 180.38472, 180.41133, 180.43765, 180.46451, 180.49187, 180.51939, 180.54758, 180.57634, 180.60477, 180.63396, 180.66389, 180.69472, 180.72603, 180.7572, 180.78957, 180.823, 180.85631, 180.88991, 180.92371, 180.95706, 180.99092, 181.02626, 181.06326, 181.10162, 181.1391, 181.17641, 181.21402, 181.25211, 181.28955, 181.32634, 181.36447, 181.40189, 181.4381, 181.47331, 181.50807, 181.54071, 181.57346, 181.60866, 181.64577, 181.68417, 181.72168, 181.75914, 181.79767, 181.83748, 181.87747, 181.91742, 181.95695, 181.99832, 182.03812, 182.07738, 182.11449, 182.15204, 182.19035, 182.22978, 182.2695, 182.31001, 182.34891, 182.38696, 182.42218, 182.45525, 182.48941, 182.52226, 182.55621, 182.58896, 182.62086, 182.65288, 182.68657, 182.72272, 182.76212, 182.80115, 182.83951, 182.87524, 182.90919, 182.94313, 182.97842, 183.01477, 183.0529, 183.09117, 183.127, 183.16306, 183.20122, 183.24178, 183.28111, 183.32036, 183.35971, 183.3998, 183.43983, 183.47787, 183.51186, 183.54558, 183.57816, 183.6123, 183.64774, 183.68333, 183.72012, 183.75874, 183.79793, 183.83867, 183.87993, 183.92157, 183.96465, 184.00539, 184.04436, 184.0843, 184.12569, 184.16653, 184.20705, 184.24741, 184.28691, 184.32756, 184.36906, 184.41148, 184.45378, 184.4951, 184.53712, 184.57993, 184.62045, 184.65775, 184.69293, 184.72659, 184.76007, 184.79503, 184.83018, 184.86899, 184.90979, 184.95056, 184.99091, 185.03053, 185.07204, 185.11502, 185.15868, 185.20329, 185.24709, 185.29115, 185.33409, 185.37717, 185.4185, 185.45804, 185.49718, 185.53632, 185.57599, 185.61728, 185.65776, 185.69963, 185.74083, 185.78281, 185.82603, 185.86871, 185.91023, 185.94936, 185.98782, 186.0262, 186.06454, 186.10416, 186.14491, 186.1852, 186.2245, 186.26433, 186.30334, 186.34256, 186.38142, 186.41753, 186.45586, 186.49515, 186.5363, 186.57649, 186.61508, 186.65221, 186.6895, 186.72816, 186.76711, 186.80779, 186.84801, 186.88885, 186.93158, 186.97491, 187.01726, 187.06096, 187.10196, 187.14183, 187.18462, 187.22882, 187.27315, 187.31848, 187.36339, 187.40767, 187.45337, 187.49886, 187.54268, 187.58609, 187.62961, 187.67044, 187.71268, 187.75528, 187.79819, 187.84183, 187.88416, 187.92462, 187.96719, 188.0098, 188.0549, 188.10202, 188.14798, 188.19414, 188.23969, 188.28632, 188.33499, 188.38423, 188.43146, 188.47794, 188.52431, 188.57013, 188.61865, 188.66565, 188.71187, 188.75861, 188.80621, 188.85393, 188.90173, 188.94839, 188.99448, 189.04036, 189.08531, 189.13077, 189.17767, 189.22517, 189.27315, 189.32074, 189.36909, 189.41704, 189.46393, 189.5119, 189.5609, 189.61021, 189.66124, 189.71246, 189.76324, 189.81259, 189.86185, 189.91013, 189.96013, 190.0108, 190.061, 190.11232, 190.1635, 190.21367, 190.2627, 190.31346, 190.36389, 190.41492, 190.46727, 190.51939, 190.57338, 190.62749, 190.68044, 190.73311, 190.78491, 190.83577, 190.8877, 190.93848, 190.98965, 191.04053, 191.09221, 191.1438, 191.19595, 191.24683, 191.29836, 191.35121, 191.40576, 191.45865, 191.51144, 191.56329, 191.61534, 191.66661, 191.71944, 191.77365, 191.82733, 191.88013, 191.93358, 191.98837, 192.04231, 192.09724, 192.15228, 192.20715, 192.26242, 192.32021, 192.37662, 192.4319, 192.48772, 192.54413, 192.59987, 192.65529, 192.71152, 192.76802, 192.82562, 192.88312, 192.94026, 192.99599, 193.05467, 193.11278, 193.17015, 193.22783, 193.28326, 193.33839, 193.39395, 193.44897, 193.50545, 193.563, 193.61928, 193.67555, 193.73364, 193.79195, 193.85016, 193.90939, 193.96805, 194.02667, 194.08534, 194.14226, 194.20026, 194.25986, 194.32065, 194.38155, 194.44293, 194.50323, 194.56407, 194.62587, 194.68752, 194.74759, 194.80595, 194.86389, 194.92307, 194.98349]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [179.94354, 179.94354, 179.94354, 179.94353, 179.94351, 179.94351, 179.9435, 179.94337, 179.94319, 179.94301, 179.94168, 179.94092, 179.94034, 179.9382, 179.93718, 179.93637, 179.93611, 179.93633, 179.93683, 179.93695, 179.93684, 179.93649, 179.9361, 179.93663, 179.93771, 179.93913, 179.94032, 179.94113, 179.94214, 179.94365, 179.94586, 179.94824, 179.95052, 179.95296, 179.95572, 179.95921, 179.96291, 179.96681, 179.97093, 179.97545, 179.98062, 179.98616, 179.99197, 179.99846, 180.00552, 180.01314, 180.02119, 180.03004, 180.0396, 180.05011, 180.06131, 180.07315, 180.08542, 180.0985, 180.11215, 180.12645, 180.14087, 180.15598, 180.17198, 180.18895, 180.20711, 180.22621, 180.24666, 180.26831, 180.28981, 180.31268, 180.33565, 180.35945, 180.38472, 180.41133, 180.43765, 180.46451, 180.49187, 180.51939, 180.54758, 180.57634, 180.60477, 180.63396, 180.66389, 180.69472, 180.72603, 180.7572, 180.78957, 180.823, 180.85631, 180.88991, 180.92371, 180.95706, 180.99092, 181.02626, 181.06326, 181.10162, 181.1391, 181.17641, 181.21402, 181.25211, 181.28955, 181.32634, 181.36447, 181.40189, 181.4381, 181.47331, 181.50807, 181.54071, 181.57346, 181.60866, 181.64577, 181.68417, 181.72168, 181.75914, 181.79767, 181.83748, 181.87747, 181.91742, 181.95695, 181.99832, 182.03812, 182.07738, 182.11449, 182.15204, 182.19035, 182.22978, 182.2695, 182.31001, 182.34891, 182.38696, 182.42218, 182.45525, 182.48941, 182.52226, 182.55621, 182.58896, 182.62086, 182.65288, 182.68657, 182.72272, 182.76212, 182.80115, 182.83951, 182.87524, 182.90919, 182.94313, 182.97842, 183.01477, 183.0529, 183.09117, 183.127, 183.16306, 183.20122, 183.24178, 183.28111, 183.32036, 183.35971, 183.3998, 183.43983, 183.47787, 183.51186, 183.54558, 183.57816, 183.6123, 183.64774, 183.68333, 183.72012, 183.75874, 183.79793, 183.83867, 183.87993, 183.92157, 183.96465, 184.00539, 184.04436, 184.0843, 184.12569, 184.16653, 184.20705, 184.24741, 184.28691, 184.32756, 184.36906, 184.41148, 184.45378, 184.4951, 184.53712, 184.57993, 184.62045, 184.65775, 184.69293, 184.72659, 184.76007, 184.79503, 184.83018, 184.86899, 184.90979, 184.95056, 184.99091, 185.03053, 185.07204, 185.11502, 185.15868, 185.20329, 185.24709, 185.29115, 185.33409, 185.37717, 185.4185, 185.45804, 185.49718, 185.53632, 185.57599, 185.61728, 185.65776, 185.69963, 185.74083, 185.78281, 185.82603, 185.86871, 185.91023, 185.94936, 185.98782, 186.0262, 186.06454, 186.10416, 186.14491, 186.1852, 186.2245, 186.26433, 186.30334, 186.34256, 186.38142, 186.41753, 186.45586, 186.49515, 186.5363, 186.57649, 186.61508, 186.65221, 186.6895, 186.72816, 186.76711, 186.80779, 186.84801, 186.88885, 186.93158, 186.97491, 187.01726, 187.06096, 187.10196, 187.14183, 187.18462, 187.22882, 187.27315, 187.31848, 187.36339, 187.40767, 187.45337, 187.49886, 187.54268, 187.58609, 187.62961, 187.67044, 187.71268, 187.75528, 187.79819, 187.84183, 187.88416, 187.92462, 187.96719, 188.0098, 188.0549, 188.10202, 188.14798, 188.19414, 188.23969, 188.28632, 188.33499, 188.38423, 188.43146, 188.47794, 188.52431, 188.57013, 188.61865, 188.66565, 188.71187, 188.75861, 188.80621, 188.85393, 188.90173, 188.94839, 188.99448, 189.04036, 189.08531, 189.13077, 189.17767, 189.22517, 189.27315, 189.32074, 189.36909, 189.41704, 189.46393, 189.5119, 189.5609, 189.61021, 189.66124, 189.71246, 189.76324, 189.81259, 189.86185, 189.91013, 189.96013, 190.0108, 190.061, 190.11232, 190.1635, 190.21367, 190.2627, 190.31346, 190.36389, 190.41492, 190.46727, 190.51939, 190.57338, 190.62749, 190.68044, 190.73311, 190.78491, 190.83577, 190.8877, 190.93848, 190.98965, 191.04053, 191.09221, 191.1438, 191.19595, 191.24683, 191.29836, 191.35121, 191.40576, 191.45865, 191.51144, 191.56329, 191.61534, 191.66661, 191.71944, 191.77365, 191.82733, 191.88013, 191.93358, 191.98837, 192.04231, 192.09724, 192.15228, 192.20715, 192.26242, 192.32021, 192.37662, 192.4319, 192.48772, 192.54413, 192.59987, 192.65529, 192.71152, 192.76802, 192.82562, 192.88312, 192.94026, 192.99599, 193.05467, 193.11278, 193.17015, 193.22783, 193.28326, 193.33839, 193.39395, 193.44897, 193.50545, 193.563, 193.61928, 193.67555, 193.73364, 193.79195, 193.85016, 193.90939, 193.96805, 194.02667, 194.08534, 194.14226, 194.20026, 194.25986, 194.32065, 194.38155, 194.44293, 194.50323, 194.56407, 194.62587, 194.68752, 194.74759, 194.80595, 194.86389, 194.92307, 194.98349]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [23.29918, 0.71187, 0.71207, 0.69449, 0.69446, 0.69443, 0.6988, 0.69196, 0.7146, 0.69983, 0.70196, 0.70471, 0.70358, 0.70105, 0.71451, 0.69917, 0.69866, 0.69442, 0.6948, 0.69086, 0.69495, 0.68836, 0.69965, 0.69226, 0.69484, 0.69875, 0.70073, 0.70246, 0.72083, 0.7009, 0.70048, 0.7008, 0.70366, 0.69412, 0.70178, 0.69908, 0.70543, 0.69424, 0.70464, 0.69955, 0.70803, 0.69841, 0.70257, 0.70418, 0.70875, 0.715, 0.70906, 0.70541, 0.71931, 0.7041, 0.70223, 0.70658, 0.69701, 0.69756, 0.69594, 0.70155, 0.70926, 0.70288, 0.6981, 0.70914, 0.69799, 0.70314, 0.70633, 0.70075, 0.70007, 0.70459, 0.70195, 0.69392, 0.7045, 0.70374, 0.70075, 0.69331, 0.69436, 0.6955, 0.70291, 0.69782, 0.70126, 0.70025, 0.70132, 0.7027, 0.70476, 0.70307, 0.69742, 0.69952, 0.69723, 0.8289, 0.70367, 0.7045, 0.70784, 0.71072, 0.70676, 0.70275, 0.70232, 0.70275, 0.70734, 0.70267, 0.70508, 0.70045, 0.70283, 0.71431, 0.708, 0.70934, 0.70749, 0.71204, 0.70839, 0.70834, 0.70947, 0.70787, 0.70812, 0.70457, 0.70563, 0.69994, 0.70262, 0.69627, 0.69863, 0.69913, 0.71178, 0.71423, 0.70926, 0.70785, 0.70607, 0.70391, 0.71582, 0.71055, 0.71123, 0.70438, 0.71121, 0.71074, 0.70765, 0.70483, 0.70686, 0.71125, 0.70564, 0.70533, 0.7078, 0.70873, 0.70986, 0.70805, 0.70797, 0.71206, 0.70956, 0.70912, 0.71021, 0.70934, 0.70819, 0.70233, 0.70414, 0.70448, 0.70564, 0.7015, 0.70586, 0.70217, 0.7129, 0.70787, 0.7092, 0.71158, 0.7112, 0.71167, 0.70869, 0.70914, 0.70573, 0.7106, 0.70502, 0.70709, 0.70454, 0.70862, 0.70342, 0.70716, 0.70517, 0.70888, 0.71242, 0.71066, 0.71063, 0.70907, 0.71159, 0.71233, 0.7117, 0.7115, 0.70892, 0.71015, 0.71212, 0.70842, 0.70856, 0.71199, 0.71305, 0.71701, 0.71312, 0.71367, 0.71284, 0.70741, 0.70964, 0.70851, 0.71466, 0.70509, 0.72116, 0.72852, 0.71403, 0.70864, 0.70955, 0.7163, 0.6926, 0.70139, 0.71844, 0.70855, 0.71025, 0.71363, 0.7113, 0.7081, 0.71651, 0.71161, 0.7088, 0.70621, 0.76558, 0.71366, 0.71465, 0.70832, 0.71501, 0.71439, 0.70996, 0.71112, 0.71318, 0.71005, 0.71114, 0.70462, 0.71021, 0.71174, 0.71118, 0.70552, 0.70941, 0.71352, 0.70296, 0.7077, 0.71087, 0.70967, 0.71319, 0.70487, 0.71314, 0.71027, 0.71726, 0.70291, 0.70583, 0.70043, 0.71003, 0.70162, 0.71159, 0.70538, 0.70772, 0.7058, 0.70393, 0.70436, 0.70523, 0.7076, 0.70951, 0.7073, 0.70677, 0.70977, 0.70523, 0.70814, 0.70619, 0.71387, 0.71394, 0.71664, 0.709, 0.70954, 0.71091, 0.71119, 0.7066, 0.71015, 0.71379, 0.70807, 0.7089, 0.70687, 0.70782, 0.70284, 0.7093, 0.70472, 0.70627, 0.70878, 0.7131, 0.71354, 0.70817, 0.7085, 0.70989, 0.7104, 0.70981, 0.70998, 0.70926, 0.70687, 0.71184, 0.7147, 0.71202, 0.70554, 0.70696, 0.71095, 0.7109, 0.70487, 0.7074, 0.70395, 0.70783, 0.70406, 0.71161, 0.70987, 0.70579, 0.70936, 0.81441, 0.70896, 0.70653, 0.70759, 0.71046, 0.70652, 0.70807, 0.70162, 0.70833, 0.70934, 0.70659, 0.71222, 0.71582, 0.71966, 0.71029, 0.70866, 0.70674, 0.71991, 0.7103, 0.70757, 0.71472, 0.70914, 0.71354, 0.8287, 0.71145, 0.70825, 0.71369, 0.71612, 0.71567, 0.71261, 0.71066, 0.70918, 0.70607, 0.70956, 0.72641, 0.7127, 0.71743, 0.70933, 0.71054, 0.70211, 0.7054, 0.70442, 0.712, 0.71222, 0.71615, 0.71003, 0.71338, 0.71009, 0.71334, 0.71107, 0.71501, 0.71714, 0.70686, 0.70974, 0.71546, 0.70423, 0.71293, 0.71055, 0.71309, 0.71563, 0.71163, 0.71034, 0.71044, 0.71, 0.70833, 0.71033, 0.70852, 0.7031, 0.71412, 0.70792, 0.71185, 0.70919, 0.7121, 0.70689, 0.71208, 0.70677, 0.7134, 0.71312, 0.71483, 0.71357, 0.71752, 0.7209, 0.71431, 0.71061, 0.71548, 0.7187, 0.71617, 0.71164, 0.71417, 0.71386, 0.71464, 0.71363, 0.71829, 0.72097, 0.71465, 0.7123]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60433]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60433]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [271.59912]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [271.59912]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/model_config.yaml
index 75184faec3..48acb1e697 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp1_pp2_fp8_pp/model_config.yaml
@@ -1,6 +1,7 @@
 ENV_VARS:
   CUDA_DEVICE_MAX_CONNECTIONS: 1
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
 MODEL_ARGS:
@@ -45,7 +46,7 @@ MODEL_ARGS:
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/golden_values.json
new file mode 100644
index 0000000000..d314392934
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [20.88514, 1.46887, 1.45698, 1.45724, 1.47204, 1.4532, 1.46049, 1.46232, 1.46114, 1.45572, 1.45278, 1.45251, 1.4606, 1.45971, 1.45327, 1.45649, 1.45387, 1.44992, 1.45853, 1.46565, 1.45437, 1.4525, 1.45638, 1.45952, 1.45173, 1.46389, 1.45431, 1.45274, 1.4583, 1.45541, 1.44989, 1.45048, 1.44894, 1.45131, 1.45345, 1.44108, 1.44133, 1.44014, 1.45925, 1.44689, 1.44677, 1.45727, 1.45173, 1.45401, 1.46616, 1.45271, 1.45499, 1.46938, 1.4604, 1.4635, 1.4619, 1.46438, 1.45747, 1.46752, 1.45729, 1.46194, 1.46122, 1.46137, 1.46148, 1.46024, 1.45382, 1.46877, 1.45937, 1.46525, 1.46624, 1.46409, 1.4727, 1.46116, 1.46451, 1.4659, 1.45827, 1.45377, 1.47607, 1.46536, 1.45984, 1.46776, 1.47935, 1.47512, 1.47012, 1.47272, 1.47499, 1.47329, 1.4585, 1.45704, 1.4555, 1.46025, 1.46072, 1.45592, 1.45507, 1.45416, 1.45424, 1.46471, 1.45308, 1.45358, 1.45797, 1.46272, 1.45587, 1.47021, 1.47373, 1.47488, 1.45879, 1.45526, 1.46684, 1.45424, 1.46048, 1.45539, 1.45476, 1.46257, 1.46204, 1.4552, 1.46046, 1.45792, 1.45501, 1.46191, 1.47519, 1.45861, 1.46195, 1.4555, 1.46541, 1.45771, 1.45708, 1.46256, 1.46253, 1.45733, 1.46154, 1.46224, 1.45714, 1.46628, 1.462, 1.46251, 1.46041, 1.45921, 1.45844, 1.46129, 1.45453, 1.45615, 1.45383, 1.45915, 1.45368, 1.46097, 1.4609, 1.4519, 1.46109, 1.45906, 1.45677, 1.46323, 1.45746, 1.45755, 1.46188, 1.45867, 1.45807, 1.45578, 1.46681, 1.46385, 1.46569, 1.4551, 1.46369, 1.45943, 1.45524, 1.45829, 1.45857, 1.45785, 1.45457, 1.44886, 1.45654, 1.4591, 1.4583, 1.46482, 1.45668, 1.45572, 1.45853, 1.46203, 1.46116, 1.45964, 1.4598, 1.46157, 1.46339, 1.45804, 1.46302, 1.4604, 1.4681, 1.4619, 1.46043, 1.46458, 1.44955, 1.45921, 1.46214, 1.45918, 1.45767, 1.45627, 1.45501, 1.46271, 1.46011, 1.45047, 1.45537, 1.45774, 1.45791, 1.45844, 1.45736, 1.45685, 1.44897, 1.46515, 1.44824, 1.4544, 1.46501, 1.45918, 1.45782, 1.45713, 1.45546, 1.4536, 1.46366, 1.45823, 1.45916, 1.45823, 1.45337, 1.46118, 1.46699, 1.4587, 1.46699, 1.47055, 1.46344, 1.46652, 1.46046, 1.46265, 1.46449, 1.46285, 1.46692, 1.45814, 1.45886, 1.46803, 1.46061, 1.45819, 1.4648, 1.46266, 1.46133, 1.46278, 1.4587, 1.46188, 1.46627, 1.45851, 1.45538, 1.46707, 1.4652, 1.45779, 1.46235, 1.45952, 1.56522, 1.45535, 1.46212, 1.53267, 1.46331, 1.56631, 1.46611, 1.4675, 1.46789, 1.46422, 1.46465, 1.46332, 1.46526, 1.46728, 1.46084, 1.46879, 1.4673, 1.46097, 1.4632, 1.46893, 1.46312, 1.47082, 1.47286, 1.46203, 1.46457, 1.46392, 1.47428, 1.46372, 1.46741, 1.46293, 1.46502, 1.46743, 1.46135, 1.45986, 1.46485, 1.45803, 1.46118, 1.46355, 1.46477, 1.4597, 1.46145, 1.46577, 1.46316, 1.46246, 1.45852, 1.46444, 1.46127, 1.46343, 1.46846, 1.46172, 1.4611, 1.46651, 1.46449, 1.45901, 1.46118, 1.46452, 1.47046, 1.46733, 1.46134, 1.4708, 1.46233, 1.46381, 1.46441, 1.47211, 1.46336, 1.46499, 1.45935, 1.46955, 1.46104, 1.46986, 1.47015, 1.46324, 1.46425, 1.46739, 1.46074, 1.46764, 1.46483, 1.46352, 1.46907, 1.4704, 1.47514, 1.4677, 1.47074, 1.46865, 1.4746, 1.47247, 1.47112, 1.47411, 1.47813, 1.47421, 1.46569, 1.46574, 1.47004, 1.46433, 1.45849, 1.46834, 1.47747, 1.46919, 1.47242, 1.46719, 1.45884, 1.462, 1.45808, 1.46357, 1.46256, 1.4583, 1.53085, 1.46007, 1.56675, 1.46277, 1.46292, 1.54903, 1.46448, 1.46847, 1.46708, 1.47477, 1.46444, 1.46433, 1.46714, 1.46403, 1.46557, 1.4607, 1.4618, 1.4615, 1.45857, 1.46496, 1.46801, 1.46664, 1.45296, 1.45665, 1.46006, 1.46236, 1.46106, 1.4622, 1.46573, 1.46166, 1.45667, 1.4563, 1.46152, 1.45678, 1.45303, 1.46242, 1.46316, 1.46041, 1.4655, 1.45096, 1.45962, 1.46428, 1.45196, 1.46789, 1.45986, 1.45627, 1.46454, 1.46424]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.36252, 0.75642, 0.75338, 0.74782, 0.75864, 0.75119, 0.75271, 0.75652, 0.75238, 0.74967, 0.74518, 0.74699, 0.74982, 0.74683, 0.74477, 0.74825, 0.75424, 0.74304, 0.74908, 0.74831, 0.74285, 0.74505, 0.75194, 0.75268, 0.74597, 0.75419, 0.74822, 0.74832, 0.75308, 0.7494, 0.74312, 0.74787, 0.74249, 0.74586, 0.74659, 0.74391, 0.7376, 0.74214, 0.75476, 0.74522, 0.74687, 0.75765, 0.7462, 0.75118, 0.75883, 0.7495, 0.7508, 0.75734, 0.7532, 0.75555, 0.75913, 0.75728, 0.75891, 0.75923, 0.75304, 0.75387, 0.75689, 0.75658, 0.76074, 0.76432, 0.75769, 0.76347, 0.75739, 0.7616, 0.76613, 0.76452, 0.76556, 0.76205, 0.76331, 0.76266, 0.7584, 0.75596, 0.77338, 0.76537, 0.75847, 0.77247, 0.7698, 0.76711, 0.76502, 0.76683, 0.76807, 0.76879, 0.75959, 0.75609, 0.7542, 0.75889, 0.7586, 0.75685, 0.75677, 0.7569, 0.75222, 0.75781, 0.74463, 0.74619, 0.75051, 0.75082, 0.74909, 0.7631, 0.75774, 0.76204, 0.75145, 0.745, 0.75456, 0.75, 0.75135, 0.75247, 0.74698, 0.7545, 0.75599, 0.74765, 0.75411, 0.75279, 0.74869, 0.75208, 0.75762, 0.74974, 0.75249, 0.74767, 0.75172, 0.74899, 0.751, 0.74685, 0.75057, 0.75145, 0.7525, 0.75608, 0.74708, 0.75458, 0.7537, 0.74712, 0.75411, 0.7543, 0.74836, 0.74769, 0.74953, 0.75136, 0.75937, 0.76403, 0.75925, 0.76123, 0.76488, 0.75935, 0.76327, 0.7569, 0.75895, 0.76622, 0.76412, 0.75914, 0.76039, 0.76442, 0.76455, 0.76016, 0.76196, 0.76613, 0.76729, 0.75679, 0.75985, 0.75945, 0.76323, 0.7635, 0.75457, 0.75811, 0.75642, 0.74425, 0.74872, 0.75503, 0.74958, 0.75606, 0.7608, 0.75663, 0.75567, 0.76176, 0.76045, 0.76145, 0.76278, 0.76702, 0.76166, 0.75954, 0.76405, 0.76075, 0.76028, 0.75744, 0.76195, 0.75996, 0.76397, 0.76843, 0.76911, 0.76882, 0.76899, 0.76126, 0.76583, 0.77184, 0.76598, 0.76126, 0.76043, 0.75584, 0.7596, 0.7606, 0.75826, 0.75896, 0.75754, 0.76441, 0.75157, 0.75476, 0.76479, 0.75674, 0.75885, 0.75822, 0.75074, 0.75763, 0.76244, 0.75885, 0.75847, 0.7616, 0.75912, 0.76519, 0.75935, 0.75886, 0.75905, 0.76846, 0.7612, 0.7615, 0.76008, 0.76429, 0.75844, 0.75869, 0.76255, 0.76097, 0.75995, 0.76319, 0.76129, 0.76036, 0.76016, 0.76111, 0.76323, 0.76537, 0.759, 0.7601, 0.76445, 0.75571, 0.75685, 0.76075, 0.75723, 0.75653, 0.75845, 0.75674, 0.86396, 0.75777, 0.76008, 0.79802, 0.76226, 0.86191, 0.76011, 0.76317, 0.76386, 0.7605, 0.76066, 0.76276, 0.76322, 0.7613, 0.7592, 0.762, 0.76075, 0.75635, 0.75896, 0.7677, 0.7624, 0.76381, 0.76676, 0.75786, 0.75925, 0.76099, 0.76684, 0.7623, 0.76206, 0.76286, 0.76089, 0.75817, 0.75534, 0.75831, 0.76571, 0.76592, 0.76306, 0.76728, 0.76327, 0.76387, 0.7666, 0.76417, 0.7663, 0.7669, 0.76023, 0.76799, 0.76358, 0.76252, 0.76815, 0.76889, 0.76519, 0.77456, 0.76596, 0.76411, 0.76815, 0.77016, 0.77392, 0.76784, 0.76277, 0.77204, 0.76778, 0.7655, 0.76653, 0.76663, 0.7655, 0.76981, 0.76378, 0.76855, 0.76427, 0.77286, 0.76279, 0.75723, 0.75876, 0.76093, 0.75608, 0.76062, 0.75705, 0.75985, 0.76693, 0.76742, 0.77256, 0.76978, 0.76789, 0.76969, 0.76933, 0.77265, 0.76608, 0.76739, 0.77128, 0.76748, 0.75765, 0.75397, 0.76206, 0.75882, 0.75813, 0.76547, 0.77479, 0.76791, 0.77465, 0.76715, 0.75994, 0.76202, 0.75688, 0.75371, 0.75879, 0.75648, 0.78313, 0.75471, 0.85298, 0.75745, 0.75629, 0.79889, 0.75755, 0.7675, 0.76401, 0.77476, 0.7623, 0.76426, 0.77061, 0.76259, 0.76592, 0.76419, 0.76322, 0.76581, 0.76288, 0.76458, 0.76887, 0.76604, 0.7592, 0.7636, 0.76038, 0.76398, 0.76433, 0.76564, 0.7642, 0.76491, 0.76122, 0.76383, 0.76659, 0.76312, 0.76135, 0.76522, 0.76474, 0.76522, 0.76449, 0.75942, 0.76396, 0.76563, 0.75814, 0.76753, 0.76464, 0.7621, 0.77007, 0.76728]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.28133, 0.68196, 0.6748, 0.67881, 0.68478, 0.67217, 0.67802, 0.67659, 0.67892, 0.67668, 0.67659, 0.67465, 0.67463, 0.67462, 0.67762, 0.67642, 0.6769, 0.67572, 0.67809, 0.68097, 0.67934, 0.67704, 0.67406, 0.67837, 0.6757, 0.67949, 0.67968, 0.6787, 0.67717, 0.68038, 0.67537, 0.67968, 0.67434, 0.67314, 0.67835, 0.66827, 0.67483, 0.66865, 0.67777, 0.67612, 0.66888, 0.68034, 0.67914, 0.67754, 0.686, 0.67891, 0.6825, 0.69249, 0.68805, 0.68071, 0.6807, 0.68401, 0.68197, 0.68831, 0.67921, 0.68344, 0.68292, 0.68269, 0.67859, 0.67491, 0.67595, 0.68683, 0.68164, 0.68009, 0.68194, 0.68378, 0.68844, 0.68048, 0.67795, 0.68343, 0.6796, 0.67682, 0.6863, 0.68552, 0.67712, 0.67901, 0.6881, 0.68205, 0.67931, 0.68414, 0.68584, 0.68259, 0.67712, 0.67748, 0.67636, 0.67686, 0.67957, 0.67669, 0.67544, 0.67461, 0.67469, 0.68134, 0.68, 0.67587, 0.68021, 0.68045, 0.67544, 0.67937, 0.68676, 0.68585, 0.67936, 0.68061, 0.68245, 0.67815, 0.67775, 0.6759, 0.67787, 0.68054, 0.6803, 0.67305, 0.67653, 0.67563, 0.67417, 0.68429, 0.68658, 0.67537, 0.68025, 0.6803, 0.68056, 0.6828, 0.68066, 0.68532, 0.67902, 0.67418, 0.68192, 0.6772, 0.6791, 0.68139, 0.68311, 0.68253, 0.67839, 0.67915, 0.67948, 0.68314, 0.67734, 0.67756, 0.67316, 0.67604, 0.6758, 0.67978, 0.67641, 0.67242, 0.67813, 0.67872, 0.6783, 0.67885, 0.67431, 0.67749, 0.67801, 0.6758, 0.67622, 0.67701, 0.68426, 0.6762, 0.67926, 0.67417, 0.68505, 0.67444, 0.67174, 0.67764, 0.67913, 0.67644, 0.67728, 0.67567, 0.67951, 0.67766, 0.67997, 0.68347, 0.67314, 0.66987, 0.67882, 0.67735, 0.67469, 0.67484, 0.67452, 0.67036, 0.67219, 0.66928, 0.67596, 0.68103, 0.68041, 0.67951, 0.67362, 0.6784, 0.6726, 0.67127, 0.67283, 0.67413, 0.67371, 0.67426, 0.67198, 0.67275, 0.67579, 0.66994, 0.67168, 0.6776, 0.67237, 0.67165, 0.67104, 0.67192, 0.67427, 0.67627, 0.66668, 0.66922, 0.67584, 0.67473, 0.6708, 0.67557, 0.67335, 0.67079, 0.67545, 0.67499, 0.67953, 0.67406, 0.67059, 0.67194, 0.67815, 0.67685, 0.67968, 0.67768, 0.67845, 0.68065, 0.67662, 0.67606, 0.68139, 0.67895, 0.67961, 0.67462, 0.67355, 0.68106, 0.67561, 0.67393, 0.67793, 0.67786, 0.6746, 0.67779, 0.67398, 0.67743, 0.67735, 0.67743, 0.67124, 0.68018, 0.68312, 0.67575, 0.67441, 0.67795, 0.77498, 0.67162, 0.6764, 0.67127, 0.67597, 0.68008, 0.68042, 0.67905, 0.68174, 0.67734, 0.68026, 0.6787, 0.67714, 0.682, 0.67394, 0.68013, 0.68188, 0.67889, 0.67722, 0.67427, 0.67656, 0.68229, 0.68021, 0.6768, 0.68025, 0.67886, 0.68439, 0.67958, 0.6764, 0.67518, 0.67551, 0.68714, 0.67915, 0.67531, 0.67638, 0.674, 0.67847, 0.67644, 0.67977, 0.674, 0.67593, 0.68097, 0.67926, 0.67773, 0.67609, 0.6796, 0.67785, 0.67882, 0.67923, 0.6747, 0.67544, 0.67361, 0.68038, 0.67547, 0.67624, 0.67248, 0.67952, 0.68043, 0.67937, 0.67985, 0.67588, 0.68025, 0.67916, 0.68539, 0.67959, 0.67855, 0.67714, 0.68454, 0.67696, 0.67981, 0.683, 0.68247, 0.6825, 0.68134, 0.67836, 0.68273, 0.68212, 0.68044, 0.67659, 0.67798, 0.67887, 0.67623, 0.67774, 0.67659, 0.67891, 0.67811, 0.68204, 0.68313, 0.68107, 0.68061, 0.68094, 0.68548, 0.68238, 0.67942, 0.67349, 0.67874, 0.67949, 0.67779, 0.67431, 0.67512, 0.67432, 0.67473, 0.67593, 0.68238, 0.67917, 0.67651, 0.68094, 0.67897, 0.68533, 0.67806, 0.68435, 0.68504, 0.682, 0.68404, 0.68368, 0.68461, 0.68091, 0.6825, 0.67628, 0.68089, 0.6828, 0.67779, 0.67875, 0.67869, 0.67726, 0.67954, 0.68441, 0.67716, 0.67303, 0.67398, 0.67541, 0.6785, 0.67881, 0.67645, 0.68188, 0.67884, 0.67565, 0.67403, 0.67785, 0.67584, 0.67366, 0.67828, 0.67909, 0.67494, 0.68175, 0.67414, 0.67764, 0.68174, 0.67366, 0.68332, 0.67954, 0.67548, 0.67937, 0.67851]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.31358, 0.01342, 0.01402, 0.01374, 0.01299, 0.01268, 0.01392, 0.01354, 0.01304, 0.01288, 0.01303, 0.01298, 0.01232, 0.01255, 0.01299, 0.01326, 0.01362, 0.0129, 0.01443, 0.01263, 0.01254, 0.01285, 0.01249, 0.01344, 0.01424, 0.01237, 0.01372, 0.01224, 0.013, 0.01253, 0.01341, 0.01286, 0.01401, 0.01393, 0.01367, 0.01532, 0.01387, 0.01392, 0.01291, 0.01426, 0.0158, 0.01586, 0.01402, 0.01614, 0.01699, 0.0155, 0.01558, 0.01634, 0.01595, 0.01549, 0.01633, 0.01561, 0.01611, 0.01605, 0.01621, 0.01402, 0.01567, 0.01545, 0.0163, 0.01651, 0.01564, 0.01603, 0.01693, 0.01689, 0.01357, 0.0139, 0.01398, 0.01321, 0.0147, 0.01234, 0.01211, 0.01284, 0.01261, 0.01263, 0.01246, 0.01271, 0.01272, 0.01352, 0.01254, 0.01474, 0.01286, 0.01466, 0.01388, 0.01269, 0.01267, 0.01231, 0.01228, 0.01211, 0.01249, 0.01199, 0.01406, 0.01239, 0.012, 0.01243, 0.01264, 0.01202, 0.01259, 0.01295, 0.01265, 0.01251, 0.01294, 0.01235, 0.01204, 0.01263, 0.01427, 0.01248, 0.01231, 0.01225, 0.01258, 0.01178, 0.01262, 0.01236, 0.01219, 0.01244, 0.01253, 0.01287, 0.01341, 0.01255, 0.01211, 0.01241, 0.01252, 0.01245, 0.01248, 0.01249, 0.01246, 0.01257, 0.01439, 0.01257, 0.01277, 0.01231, 0.01239, 0.01246, 0.01285, 0.01264, 0.01226, 0.01308, 0.01475, 0.01426, 0.01226, 0.01234, 0.0128, 0.01255, 0.01327, 0.01286, 0.01198, 0.0126, 0.01182, 0.01221, 0.01291, 0.01266, 0.0138, 0.01491, 0.01556, 0.01521, 0.01547, 0.01523, 0.01535, 0.01539, 0.01545, 0.01502, 0.01553, 0.01548, 0.01523, 0.0158, 0.0149, 0.01554, 0.01524, 0.01563, 0.01495, 0.01509, 0.01539, 0.01542, 0.01541, 0.01496, 0.0133, 0.01391, 0.01409, 0.01274, 0.01438, 0.01341, 0.01299, 0.01457, 0.0135, 0.01472, 0.01228, 0.01294, 0.01287, 0.01243, 0.01296, 0.01232, 0.0131, 0.01254, 0.01253, 0.01203, 0.01548, 0.01457, 0.01673, 0.01491, 0.01608, 0.01713, 0.20109, 0.01559, 0.01542, 0.01587, 0.01537, 0.01617, 0.01548, 0.01476, 0.01531, 0.01468, 0.01359, 0.01328, 0.01334, 0.01271, 0.01326, 0.01281, 0.01274, 0.01235, 0.01343, 0.01378, 0.01234, 0.01331, 0.01322, 0.01409, 0.01395, 0.01384, 0.01454, 0.01599, 0.01706, 0.01595, 0.01555, 0.01494, 0.01652, 0.01668, 0.01556, 0.01656, 0.01651, 0.01523, 0.01549, 0.01748, 0.0151, 0.01561, 0.01593, 0.01703, 0.01695, 0.01519, 0.11815, 0.01383, 0.01413, 0.01352, 0.0127, 0.01447, 0.01336, 0.0136, 0.0135, 0.01283, 0.01313, 0.01327, 0.01457, 0.0137, 0.01312, 0.01422, 0.01356, 0.01359, 0.01298, 0.01365, 0.01348, 0.01345, 0.01333, 0.01313, 0.01267, 0.01374, 0.01318, 0.01263, 0.01428, 0.01505, 0.01249, 0.01321, 0.01297, 0.01239, 0.01264, 0.01257, 0.01217, 0.0122, 0.0122, 0.01198, 0.0127, 0.01478, 0.01247, 0.01244, 0.01216, 0.0125, 0.01376, 0.01279, 0.01258, 0.01297, 0.01503, 0.01572, 0.01498, 0.01367, 0.01289, 0.01246, 0.01343, 0.01425, 0.01243, 0.01244, 0.0128, 0.01271, 0.01294, 0.01314, 0.01241, 0.01281, 0.01413, 0.01267, 0.01236, 0.01278, 0.01212, 0.01253, 0.01258, 0.01307, 0.0136, 0.01249, 0.0128, 0.01213, 0.01404, 0.01391, 0.01279, 0.0132, 0.01312, 0.01257, 0.01296, 0.01486, 0.01348, 0.01408, 0.01312, 0.01352, 0.01264, 0.01361, 0.01373, 0.01287, 0.01447, 0.01273, 0.0134, 0.01256, 0.01471, 0.01292, 0.01296, 0.01556, 0.01269, 0.01275, 0.01262, 0.01243, 0.01254, 0.01292, 0.01389, 0.01214, 0.01259, 0.01322, 0.01252, 0.01284, 0.01326, 0.01406, 0.01221, 0.01209, 0.01445, 0.01235, 0.01243, 0.01521, 0.01303, 0.01308, 0.01361, 0.01255, 0.01227, 0.01283, 0.01623, 0.01515, 0.01582, 0.01716, 0.01637, 0.01737, 0.01732, 0.01611, 0.01683, 0.01561, 0.01502, 0.01608, 0.015, 0.01699, 0.017, 0.0159, 0.01671, 0.016, 0.01726, 0.01765, 0.01553, 0.01619, 0.01499, 0.01559, 0.01568, 0.01579]}, "forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [5.69523, 0.02394, 0.02348, 0.02329, 0.02364, 0.02293, 0.02376, 0.0234, 0.02371, 0.02468, 0.02324, 0.02396, 0.02501, 0.0256, 0.02468, 0.02408, 0.02484, 0.02364, 0.02322, 0.02328, 0.02362, 0.02407, 0.02284, 0.02422, 0.02402, 0.02397, 0.0233, 0.02317, 0.0238, 0.02388, 0.02326, 0.02363, 0.02416, 0.02354, 0.02309, 0.02365, 0.02345, 0.02308, 0.02317, 0.02313, 0.02335, 0.023, 0.02326, 0.0233, 0.0238, 0.02375, 0.02493, 0.02394, 0.02412, 0.0238, 0.02339, 0.02351, 0.02335, 0.0266, 0.0234, 0.02405, 0.02373, 0.0237, 0.02385, 0.02378, 0.02359, 0.02689, 0.02333, 0.02338, 0.02322, 0.02354, 0.0233, 0.02329, 0.02452, 0.02693, 0.02345, 0.02326, 0.02375, 0.02341, 0.02388, 0.0233, 0.02333, 0.02476, 0.02365, 0.0236, 0.02356, 0.02344, 0.02363, 0.02334, 0.0233, 0.02313, 0.02387, 0.02342, 0.02362, 0.02319, 0.02461, 0.02359, 0.0234, 0.02397, 0.02524, 0.02331, 0.02386, 0.02533, 0.02416, 0.02445, 0.02309, 0.02381, 0.02352, 0.02393, 0.02341, 0.02313, 0.02371, 0.02364, 0.02387, 0.02355, 0.02449, 0.02408, 0.02363, 0.02317, 0.02331, 0.0239, 0.02385, 0.0235, 0.02309, 0.0239, 0.02371, 0.0232, 0.0236, 0.0237, 0.0241, 0.02434, 0.02347, 0.02522, 0.02461, 0.02418, 0.02376, 0.02318, 0.02386, 0.02379, 0.02334, 0.02333, 0.02452, 0.02365, 0.02364, 0.02368, 0.02399, 0.02426, 0.02355, 0.02382, 0.02423, 0.02653, 0.02379, 0.02327, 0.02414, 0.02462, 0.02631, 0.02476, 0.02402, 0.02578, 0.02427, 0.02403, 0.02365, 0.02467, 0.02569, 0.02364, 0.02413, 0.02503, 0.02507, 0.02438, 0.02416, 0.02449, 0.02518, 0.02522, 0.02409, 0.02476, 0.02466, 0.02482, 0.02437, 0.02418, 0.0241, 0.02501, 0.02478, 0.02401, 0.02483, 0.02545, 0.02468, 0.02391, 0.02507, 0.02466, 0.02414, 0.02353, 0.0242, 0.02477, 0.02356, 0.02431, 0.02316, 0.02439, 0.02399, 0.02385, 0.02354, 0.02465, 0.02547, 0.02508, 0.02419, 0.02477, 0.01768, 0.02429, 0.02356, 0.02577, 0.02434, 0.02473, 0.02445, 0.02378, 0.02439, 0.02389, 0.02352, 0.02408, 0.02328, 0.02452, 0.02367, 0.02386, 0.02413, 0.02431, 0.02462, 0.02369, 0.02376, 0.02491, 0.02439, 0.02403, 0.02377, 0.02464, 0.02435, 0.02348, 0.02371, 0.0252, 0.02368, 0.02387, 0.02399, 0.02427, 0.02729, 0.02472, 0.02405, 0.02401, 0.02437, 0.02492, 0.02402, 0.02449, 0.02457, 0.02418, 0.02405, 0.02463, 0.02494, 0.02411, 0.02427, 0.02434, 0.02507, 0.02381, 0.02365, 0.02529, 0.02396, 0.02466, 0.0235, 0.02361, 0.02374, 0.02465, 0.02472, 0.02388, 0.02377, 0.02493, 0.02356, 0.02375, 0.024, 0.02421, 0.02437, 0.02348, 0.02314, 0.02411, 0.02461, 0.02389, 0.0247, 0.02407, 0.0246, 0.02474, 0.02412, 0.02434, 0.02469, 0.02369, 0.02397, 0.02513, 0.02411, 0.02363, 0.02383, 0.02511, 0.02474, 0.02401, 0.02392, 0.0241, 0.02386, 0.02404, 0.02408, 0.02406, 0.02452, 0.02544, 0.02797, 0.0258, 0.02429, 0.02521, 0.02549, 0.02471, 0.02437, 0.02521, 0.02445, 0.0245, 0.0237, 0.02743, 0.02449, 0.02397, 0.02369, 0.02461, 0.02423, 0.02547, 0.02366, 0.02466, 0.02473, 0.02447, 0.02511, 0.02472, 0.02518, 0.02397, 0.02404, 0.02493, 0.02555, 0.02496, 0.02436, 0.02395, 0.02507, 0.02456, 0.0243, 0.02385, 0.02539, 0.02483, 0.02431, 0.02399, 0.02469, 0.0254, 0.02512, 0.03429, 0.0364, 0.03571, 0.03561, 0.03474, 0.02415, 0.02604, 0.02499, 0.02494, 0.0246, 0.02567, 0.02501, 0.02468, 0.02397, 0.02793, 0.02468, 0.02491, 0.02539, 0.02409, 0.02475, 0.02441, 0.02562, 0.02394, 0.02557, 0.02449, 0.02381, 0.02425, 0.02474, 0.02431, 0.02389, 0.02357, 0.02526, 0.0266, 0.02574, 0.02347, 0.02485, 0.02498, 0.02413, 0.02387, 0.02515, 0.02481, 0.02439, 0.02404, 0.02457, 0.02585, 0.02502, 0.02382, 0.02429, 0.02509, 0.02444, 0.02418, 0.02439, 0.02469, 0.0242, 0.0249, 0.02556, 0.0254, 0.02589, 0.02426]}, "forward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.90859, 0.00013, 0.00013, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00041, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00011, 0.00013, 0.00011, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00011, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00011, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00014, 0.00017, 0.00016, 0.00012, 0.00017, 0.00011, 0.00012, 0.00012, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00014, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00013, 0.00013]}, "backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02368, 0.02348, 0.02394, 0.02364, 0.02449, 0.02409, 0.02505, 0.02374, 0.02528, 0.0259, 0.02358, 0.0242, 0.02637, 0.02354, 0.0251, 0.02307, 0.02342, 0.02386, 0.02487, 0.02353, 0.02241, 0.02358, 0.02336, 0.02385, 0.02423, 0.02362, 0.02431, 0.02368, 0.02447, 0.02388, 0.02278, 0.02395, 0.02289, 0.02372, 0.0236, 0.02367, 0.02368, 0.02432, 0.02399, 0.02338, 0.02355, 0.02343, 0.02344, 0.02565, 0.02464, 0.02367, 0.02563, 0.02365, 0.02498, 0.02382, 0.02437, 0.02419, 0.02505, 0.02388, 0.02389, 0.02396, 0.02377, 0.02399, 0.02396, 0.02304, 0.02377, 0.02724, 0.02399, 0.02408, 0.02416, 0.02465, 0.02583, 0.02394, 0.02408, 0.02617, 0.02288, 0.02529, 0.0259, 0.02468, 0.02405, 0.02424, 0.02366, 0.02431, 0.02501, 0.02416, 0.02392, 0.02398, 0.02395, 0.02361, 0.02493, 0.02419, 0.02355, 0.02345, 0.02429, 0.02305, 0.02433, 0.02418, 0.02434, 0.02361, 0.02432, 0.02418, 0.0234, 0.02415, 0.02349, 0.02463, 0.02416, 0.02344, 0.02561, 0.02358, 0.02435, 0.024, 0.02522, 0.02503, 0.02562, 0.02467, 0.02425, 0.02421, 0.02382, 0.0242, 0.02401, 0.02416, 0.02588, 0.0247, 0.02434, 0.02473, 0.02524, 0.02511, 0.02494, 0.02375, 0.02595, 0.02432, 0.02337, 0.02414, 0.02486, 0.0245, 0.02433, 0.02431, 0.02365, 0.02411, 0.02342, 0.02427, 0.02467, 0.02469, 0.02352, 0.02452, 0.02337, 0.02463, 0.02478, 0.02463, 0.02462, 0.02668, 0.02409, 0.02498, 0.02302, 0.02351, 0.02626, 0.02404, 0.02319, 0.02423, 0.02437, 0.02371, 0.02423, 0.02372, 0.02372, 0.02417, 0.02394, 0.02401, 0.02428, 0.02406, 0.02443, 0.02396, 0.02341, 0.02439, 0.02392, 0.02389, 0.02372, 0.02654, 0.02468, 0.02413, 0.02396, 0.02411, 0.02434, 0.02436, 0.02416, 0.02432, 0.02413, 0.02462, 0.0275, 0.02423, 0.02396, 0.027, 0.02446, 0.02452, 0.025, 0.02481, 0.02389, 0.02952, 0.02408, 0.02468, 0.02725, 0.02317, 0.02402, 0.02623, 0.02326, 0.02418, 0.0249, 0.0242, 0.02443, 0.02409, 0.0256, 0.02406, 0.02355, 0.02409, 0.02372, 0.02539, 0.02507, 0.02461, 0.02483, 0.02426, 0.02423, 0.02431, 0.02427, 0.02447, 0.02382, 0.02564, 0.02441, 0.02556, 0.02403, 0.02573, 0.02428, 0.02401, 0.02513, 0.02382, 0.02364, 0.02454, 0.02477, 0.02397, 0.0253, 0.02422, 0.02361, 0.02617, 0.02493, 0.02542, 0.0241, 0.02392, 0.02412, 0.02369, 0.02392, 0.02434, 0.02381, 0.02437, 0.02629, 0.02397, 0.0244, 0.02457, 0.02396, 0.02392, 0.02359, 0.02513, 0.02438, 0.02434, 0.02525, 0.02462, 0.02406, 0.02675, 0.0243, 0.02493, 0.02442, 0.02465, 0.02474, 0.02404, 0.02508, 0.02549, 0.02338, 0.02287, 0.02444, 0.02513, 0.02493, 0.02474, 0.0248, 0.02431, 0.0245, 0.02863, 0.02409, 0.02427, 0.02391, 0.02367, 0.02441, 0.02399, 0.02425, 0.02368, 0.0241, 0.02393, 0.02417, 0.02474, 0.02369, 0.02638, 0.02436, 0.02611, 0.02434, 0.02576, 0.02383, 0.02442, 0.02353, 0.02419, 0.02477, 0.02466, 0.02579, 0.02455, 0.0242, 0.02475, 0.02338, 0.02403, 0.02538, 0.02364, 0.02364, 0.02423, 0.02324, 0.02408, 0.02434, 0.02456, 0.0243, 0.02403, 0.02448, 0.02338, 0.02413, 0.02447, 0.02323, 0.02365, 0.02506, 0.02554, 0.02565, 0.02416, 0.025, 0.02532, 0.02482, 0.02683, 0.02458, 0.02498, 0.02491, 0.02422, 0.0243, 0.02428, 0.02417, 0.02376, 0.02431, 0.02339, 0.02362, 0.02365, 0.02371, 0.02421, 0.02393, 0.02386, 0.02374, 0.0249, 0.02454, 0.02401, 0.02418, 0.02411, 0.02461, 0.02418, 0.02303, 0.02369, 0.02384, 0.02685, 0.02364, 0.02436, 0.02417, 0.02486, 0.02423, 0.02448, 0.02462, 0.02366, 0.02415, 0.02421, 0.0243, 0.02378, 0.02574, 0.02403, 0.02374, 0.02434, 0.02432, 0.02579, 0.02343, 0.02354, 0.02396, 0.02392, 0.02373, 0.02416, 0.02348, 0.02355, 0.02427, 0.0252, 0.02486, 0.02405, 0.02393, 0.0234, 0.02443, 0.02418, 0.02422, 0.02504, 0.02408, 0.0243, 0.02762, 0.02382]}, "backward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00016, 0.00016, 0.00019, 0.00016, 0.00017, 0.00018, 0.00016, 0.00016, 0.00019, 0.00016, 0.00017, 0.00016, 0.00016, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00019, 0.00016, 0.00018, 0.00019, 0.00018, 0.00015, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00016, 0.00017, 0.00019, 0.00016, 0.00017, 0.00017, 0.00016, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00017, 0.00016, 0.00018, 0.00016, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00016, 0.00017, 0.00016, 0.00018, 0.00018, 0.00018, 0.00017, 0.00017, 0.00017, 0.00018, 0.00016, 0.00018, 0.00018, 0.00019, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00019, 0.00016, 0.00018, 0.00016, 0.00017, 0.00016, 0.00018, 0.00016, 0.00018, 0.00016, 0.00017, 0.00032, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00019, 0.00016, 0.00016, 0.00016, 0.00016, 0.00018, 0.00019, 0.00016, 0.00016, 0.00016, 0.00017, 0.00019, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.00017, 0.00016, 0.00016, 0.00018, 0.00016, 0.00018, 0.00017, 0.00016, 0.00017, 0.00025, 0.00016, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00018, 0.00017, 0.00019, 0.00016, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00031, 0.00016, 0.00016, 0.00025, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00017, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00022, 0.00016, 0.00016, 0.00016, 0.00018, 0.00016, 0.00017, 0.00017, 0.00015, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.00019, 0.00017, 0.00017, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00015, 0.00016, 0.00017, 0.00016, 0.00016, 0.00017, 0.00016, 0.00017, 0.00018, 0.00017, 0.00018, 0.00018, 0.00016, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00017, 0.00017, 0.00018, 0.00018, 0.00016, 0.00017, 0.00017, 0.00016, 0.00017, 0.00019, 0.00019, 0.00028, 0.00017, 0.00017, 0.00016, 0.00016, 0.00016, 0.00016, 0.00015, 0.00017, 0.00016, 0.00016, 0.00017, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.0002, 0.00016, 0.00017, 0.00017, 0.00018, 0.00018, 0.00016, 0.00016, 0.00017, 0.00018, 0.00018, 0.00016, 0.00023, 0.00016, 0.00016, 0.00016, 0.00017, 0.00016, 0.00019, 0.00017, 0.00016, 0.00016, 0.00015, 0.00016, 0.00018, 0.00019, 0.00016, 0.00018, 0.00017, 0.00016, 0.00017, 0.00018, 0.00018, 0.00022, 0.00016, 0.00016, 0.0002, 0.00019, 0.00017, 0.00016, 0.00018, 0.00016, 0.00016, 0.00017, 0.00016, 0.00017, 0.00019, 0.00016, 0.00016, 0.00018, 0.00017, 0.00018, 0.00015, 0.00016, 0.00016, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00016, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00017, 0.00022, 0.00016, 0.00017, 0.00016, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00017, 0.00016, 0.00017, 0.00016, 0.00026, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.00016, 0.00016, 0.00019, 0.00017, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00018, 0.00019, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00018, 0.00019, 0.00016, 0.00018, 0.00018, 0.00016, 0.00017, 0.00018, 0.00031, 0.00018, 0.00017, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00017, 0.00016, 0.00016, 0.00017, 0.00016, 0.00016, 0.00019, 0.00018, 0.00016, 0.00017, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00019, 0.00016, 0.00019]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [7.32739, 0.12477, 0.12666, 0.128, 0.12835, 0.12967, 0.1275, 0.13153, 0.12112, 0.12816, 0.12128, 0.1203, 0.12267, 0.122, 0.12207, 0.1236, 0.12689, 0.12116, 0.11515, 0.1236, 0.11731, 0.11801, 0.12855, 0.12095, 0.12421, 0.12165, 0.12224, 0.11784, 0.12171, 0.11872, 0.11626, 0.12467, 0.1241, 0.11907, 0.11776, 0.12636, 0.11891, 0.12432, 0.12301, 0.12655, 0.12996, 0.13374, 0.12156, 0.12801, 0.13689, 0.1275, 0.13219, 0.13231, 0.13041, 0.12833, 0.13716, 0.13099, 0.1317, 0.1252, 0.12341, 0.12286, 0.12995, 0.12336, 0.13226, 0.13381, 0.12738, 0.13598, 0.13071, 0.13531, 0.14271, 0.14199, 0.13871, 0.142, 0.14001, 0.14332, 0.13666, 0.13328, 0.14543, 0.14315, 0.13564, 0.15173, 0.14153, 0.15109, 0.14782, 0.14157, 0.14168, 0.14516, 0.13449, 0.13595, 0.13466, 0.13854, 0.13617, 0.13542, 0.13551, 0.13682, 0.13396, 0.13632, 0.12977, 0.13179, 0.13436, 0.12818, 0.1318, 0.15065, 0.14138, 0.14121, 0.12829, 0.1243, 0.12753, 0.13425, 0.13136, 0.13043, 0.12709, 0.1367, 0.13831, 0.13249, 0.13782, 0.13352, 0.13464, 0.12973, 0.1292, 0.13364, 0.13332, 0.13424, 0.12997, 0.13345, 0.12818, 0.13196, 0.13345, 0.13333, 0.13254, 0.13659, 0.13184, 0.13348, 0.12597, 0.13454, 0.13192, 0.1375, 0.13257, 0.12337, 0.1345, 0.13062, 0.13753, 0.13119, 0.13426, 0.13825, 0.13839, 0.13388, 0.13726, 0.12898, 0.13377, 0.13935, 0.1381, 0.13416, 0.13521, 0.13765, 0.1373, 0.13402, 0.12531, 0.13371, 0.14559, 0.13302, 0.12679, 0.13579, 0.1348, 0.13764, 0.13247, 0.13464, 0.13235, 0.13117, 0.12868, 0.13327, 0.13496, 0.1324, 0.13728, 0.13904, 0.13275, 0.14304, 0.14323, 0.14887, 0.14315, 0.1468, 0.14026, 0.14574, 0.14975, 0.14342, 0.14555, 0.13943, 0.1403, 0.1444, 0.14205, 0.14177, 0.1462, 0.14686, 0.14634, 0.14245, 0.14549, 0.14618, 0.14887, 0.13512, 0.13541, 0.13381, 0.14182, 0.14007, 0.14152, 0.13605, 0.13807, 0.13717, 0.13509, 0.13546, 0.13698, 0.13358, 0.13623, 0.13205, 0.12316, 0.13181, 0.14145, 0.1317, 0.13396, 0.14106, 0.13611, 0.14089, 0.14373, 0.13469, 0.1384, 0.14246, 0.13291, 0.14068, 0.13738, 0.13421, 0.13749, 0.13088, 0.13458, 0.13609, 0.133, 0.14241, 0.13922, 0.13388, 0.14182, 0.13246, 0.13971, 0.14107, 0.13164, 0.13039, 0.13705, 0.12577, 0.13184, 0.13088, 0.13144, 0.13487, 0.13555, 0.12695, 0.23517, 0.1322, 0.13486, 0.16077, 0.13981, 0.23534, 0.13332, 0.13076, 0.13464, 0.12966, 0.13057, 0.13577, 0.13162, 0.12711, 0.13253, 0.13694, 0.13253, 0.1291, 0.13231, 0.13615, 0.13278, 0.13306, 0.13739, 0.13635, 0.12928, 0.12884, 0.13997, 0.13381, 0.13621, 0.14094, 0.1347, 0.13224, 0.13078, 0.1333, 0.14059, 0.13768, 0.13345, 0.1394, 0.13204, 0.13595, 0.14267, 0.13406, 0.13447, 0.13958, 0.13493, 0.13657, 0.13256, 0.13241, 0.14205, 0.13985, 0.13748, 0.14438, 0.14105, 0.13704, 0.14125, 0.13958, 0.1371, 0.13476, 0.13221, 0.14116, 0.1413, 0.13323, 0.13777, 0.13451, 0.13785, 0.13827, 0.13489, 0.13565, 0.13632, 0.14132, 0.13954, 0.13567, 0.13798, 0.1411, 0.13641, 0.1346, 0.13417, 0.13059, 0.14076, 0.14564, 0.14703, 0.14826, 0.14723, 0.14169, 0.14389, 0.14245, 0.14606, 0.1389, 0.14429, 0.14006, 0.13171, 0.13461, 0.13482, 0.14111, 0.13415, 0.14396, 0.15035, 0.14874, 0.1481, 0.14804, 0.13867, 0.14775, 0.13614, 0.13103, 0.13832, 0.13379, 0.15425, 0.1329, 0.22576, 0.13539, 0.12996, 0.16565, 0.12569, 0.12696, 0.12758, 0.13901, 0.13127, 0.13219, 0.13915, 0.13046, 0.12996, 0.1351, 0.13312, 0.13428, 0.13394, 0.13287, 0.13398, 0.13368, 0.12682, 0.13561, 0.13323, 0.1307, 0.13416, 0.13272, 0.13142, 0.136, 0.13057, 0.13073, 0.13345, 0.13692, 0.13433, 0.13536, 0.13216, 0.13483, 0.13431, 0.13132, 0.13241, 0.13481, 0.13004, 0.13405, 0.12911, 0.13104, 0.13208, 0.13389]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.85465, 0.00835, 0.00699, 0.00741, 0.00706, 0.00797, 0.0072, 0.00701, 0.00796, 0.0097, 0.00702, 0.00774, 0.00734, 0.00774, 0.0089, 0.00828, 0.00699, 0.00781, 0.00859, 0.00782, 0.00885, 0.00849, 0.00699, 0.00689, 0.00726, 0.00698, 0.00708, 0.00765, 0.00904, 0.00754, 0.00764, 0.00719, 0.00699, 0.00717, 0.00867, 0.00723, 0.00713, 0.00719, 0.00696, 0.00695, 0.0071, 0.00724, 0.00738, 0.00696, 0.00708, 0.00738, 0.00771, 0.00745, 0.00704, 0.00878, 0.00742, 0.00713, 0.00774, 0.00714, 0.00691, 0.01011, 0.00831, 0.00755, 0.00829, 0.00713, 0.00712, 0.00776, 0.00714, 0.00703, 0.00812, 0.00754, 0.00844, 0.00686, 0.00703, 0.00718, 0.00709, 0.00784, 0.00743, 0.00744, 0.00705, 0.00773, 0.0077, 0.00752, 0.00823, 0.00721, 0.00697, 0.00777, 0.00754, 0.00704, 0.00687, 0.00767, 0.00697, 0.00724, 0.0081, 0.0081, 0.00692, 0.00799, 0.00739, 0.00705, 0.00849, 0.00694, 0.00742, 0.00767, 0.00711, 0.00824, 0.00696, 0.00742, 0.00848, 0.00758, 0.00786, 0.00691, 0.00711, 0.00709, 0.00692, 0.00764, 0.00779, 0.00699, 0.00727, 0.00768, 0.007, 0.0078, 0.00701, 0.00735, 0.00759, 0.00875, 0.00792, 0.00727, 0.00737, 0.00715, 0.00787, 0.00741, 0.00751, 0.00855, 0.00692, 0.00786, 0.00751, 0.00811, 0.00715, 0.00699, 0.00709, 0.00705, 0.00737, 0.0082, 0.00828, 0.00883, 0.00777, 0.00806, 0.00752, 0.0074, 0.00758, 0.00764, 0.00798, 0.00876, 0.0073, 0.00773, 0.00824, 0.00728, 0.00773, 0.00775, 0.00706, 0.00716, 0.00698, 0.00735, 0.00857, 0.00716, 0.00715, 0.00888, 0.00742, 0.00709, 0.00773, 0.00707, 0.00785, 0.00751, 0.00723, 0.00781, 0.00732, 0.00731, 0.00751, 0.00926, 0.00734, 0.00835, 0.00815, 0.00834, 0.00863, 0.00698, 0.00697, 0.00866, 0.00749, 0.00697, 0.00797, 0.00761, 0.00705, 0.00898, 0.00815, 0.00711, 0.00733, 0.00846, 0.00756, 0.00807, 0.00707, 0.00876, 0.00728, 0.00798, 0.00766, 0.00737, 0.00998, 0.00838, 0.0077, 0.00751, 0.00848, 0.00695, 0.00705, 0.00981, 0.00734, 0.00923, 0.0071, 0.00714, 0.00728, 0.00728, 0.0085, 0.00981, 0.00871, 0.00696, 0.00863, 0.00936, 0.01089, 0.00793, 0.00711, 0.00971, 0.00701, 0.00936, 0.00758, 0.00816, 0.00884, 0.00803, 0.00847, 0.01006, 0.00978, 0.00825, 0.0081, 0.00787, 0.00813, 0.00997, 0.00754, 0.00893, 0.00765, 0.00713, 0.0078, 0.0076, 0.00705, 0.00918, 0.11069, 0.00794, 0.00727, 0.07524, 0.00865, 0.00813, 0.007, 0.00696, 0.0071, 0.00698, 0.00706, 0.00709, 0.00901, 0.00738, 0.00798, 0.00783, 0.00755, 0.00757, 0.00792, 0.0078, 0.00758, 0.00842, 0.00991, 0.00945, 0.00712, 0.00835, 0.00735, 0.00734, 0.00709, 0.00708, 0.00953, 0.00709, 0.00704, 0.00922, 0.00937, 0.00856, 0.00712, 0.00846, 0.01121, 0.00908, 0.00701, 0.01037, 0.00813, 0.00814, 0.00709, 0.00791, 0.0074, 0.00756, 0.00813, 0.00849, 0.00705, 0.00877, 0.00705, 0.00702, 0.00784, 0.00699, 0.00862, 0.00977, 0.0078, 0.00851, 0.00917, 0.00814, 0.00962, 0.0071, 0.00832, 0.01014, 0.00711, 0.00716, 0.00781, 0.00825, 0.01002, 0.00758, 0.00695, 0.01037, 0.00713, 0.0097, 0.00977, 0.00754, 0.00863, 0.00703, 0.00781, 0.00826, 0.00731, 0.00742, 0.00778, 0.00814, 0.00835, 0.00713, 0.00837, 0.0071, 0.00718, 0.00856, 0.00694, 0.00858, 0.00741, 0.00763, 0.00727, 0.00894, 0.00892, 0.0078, 0.00875, 0.00972, 0.00704, 0.00701, 0.00812, 0.00733, 0.0694, 0.00715, 0.09935, 0.00722, 0.00697, 0.0823, 0.00708, 0.00762, 0.00706, 0.00717, 0.00712, 0.0071, 0.00708, 0.00694, 0.00712, 0.00717, 0.00703, 0.00723, 0.00767, 0.007, 0.00705, 0.00716, 0.00837, 0.00992, 0.00743, 0.0076, 0.00795, 0.00785, 0.00774, 0.00828, 0.00864, 0.00714, 0.00767, 0.00727, 0.0089, 0.00821, 0.00781, 0.00855, 0.00777, 0.00721, 0.00716, 0.00875, 0.00792, 0.00919, 0.00807, 0.00884, 0.00881, 0.0088]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 3e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00055, 0.00031, 0.00031, 0.00031, 0.00035, 0.00031, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00031, 0.00031, 0.00034, 0.00031, 0.00031, 0.00031, 0.00036, 0.00031, 0.00031, 0.00031, 0.00035, 0.00032, 0.00035, 0.00032, 0.00031, 0.00034, 0.00036, 0.00032, 0.00033, 0.00033, 0.00032, 0.00032, 0.00036, 0.00036, 0.00036, 0.00036, 0.00031, 0.00034, 0.00036, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00036, 0.00032, 0.00031, 0.00032, 0.00036, 0.00032, 0.00032, 0.00036, 0.00036, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00035, 0.00032, 0.00032, 0.00031, 0.00033, 0.00032, 0.00032, 0.00031, 0.00031, 0.00036, 0.00032, 0.00031, 0.00032, 0.00033, 0.00036, 0.00031, 0.00037, 0.00032, 0.00035, 0.00032, 0.00031, 0.00035, 0.00036, 0.00032, 0.00031, 0.00032, 0.00036, 0.00031, 0.00032, 0.00036, 0.00031, 0.00034, 0.00031, 0.00032, 0.00032, 0.00031, 0.00036, 0.00032, 0.00036, 0.00031, 0.00037, 0.00032, 0.00037, 0.0004, 0.00031, 0.00032, 0.00035, 0.00031, 0.00032, 0.00031, 0.00031, 0.00031, 0.00032, 0.00031, 0.00033, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00036, 0.00031, 0.00031, 0.00033, 0.00036, 0.00031, 0.00032, 0.00032, 0.00032, 0.00036, 0.00031, 0.00035, 0.00032, 0.00039, 0.00033, 0.00032, 0.00031, 0.00035, 0.00032, 0.00031, 0.00032, 0.00035, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00031, 0.00034, 0.00036, 0.00036, 0.00031, 0.00032, 0.00032, 0.00031, 0.00035, 0.00036, 0.00032, 0.00031, 0.00032, 0.00032, 0.00036, 0.00033, 0.00035, 0.00031, 0.00031, 0.00031, 0.00032, 0.00036, 0.00037, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00036, 0.00037, 0.00031, 0.00032, 0.00033, 0.00032, 0.00032, 0.00045, 0.00031, 0.00031, 0.00038, 0.00032, 0.00036, 0.00034, 0.00031, 0.00032, 0.00036, 0.00032, 0.00031, 0.00036, 0.00031, 0.00031, 0.00031, 0.00036, 0.00031, 0.00032, 0.00032, 0.0004, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00037, 0.00031, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00036, 0.00032, 0.00035, 0.00032, 0.00036, 0.00038, 0.00036, 0.00036, 0.00032, 0.00036, 0.00033, 0.00032, 0.00032, 0.00031, 0.00036, 0.00031, 0.00033, 0.00033, 0.00032, 0.00037, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00037, 0.00032, 0.00031, 0.00032, 0.00032, 0.00036, 0.00032, 0.00033, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00036, 0.00032, 0.00032, 0.00037, 0.00031, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00037, 0.00035, 0.00036, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00036, 0.00032, 0.00031, 0.00032, 0.00036, 0.00032, 0.00032, 0.00032, 0.00036, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00033, 0.00038, 0.00034, 0.00036, 0.00032, 0.00033, 0.00032, 0.00032, 0.00035, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00035, 0.00032, 0.00032, 0.00031, 0.00032, 0.00036, 0.00036, 0.00032, 0.00032, 0.00032, 0.00036, 0.00032, 0.00032, 0.00031, 0.00036, 0.00032, 0.00036, 0.00033, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00036, 0.00035, 0.00031, 0.00032, 0.00036, 0.00032, 0.00033, 0.00036, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00033, 0.00032, 0.00031, 0.00032, 0.00035, 0.00032, 0.00032, 0.00035, 0.00032, 0.00035, 0.00032, 0.00037, 0.00032, 0.00031, 0.00037, 0.00032, 0.00035, 0.00031, 0.00036, 0.00032]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.11402, 0.00057, 0.00063, 0.00057, 0.00058, 0.00057, 0.00058, 0.00058, 0.00057, 0.00063, 0.00057, 0.00058, 0.00058, 0.00057, 0.00057, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00066, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.0006, 0.00059, 0.00059, 0.00063, 0.00059, 0.00058, 0.00058, 0.00059, 0.00063, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.0006, 0.00058, 0.00058, 0.00058, 0.00057, 0.0007, 0.00059, 0.00064, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00061, 0.00058, 0.00064, 0.00058, 0.00059, 0.00059, 0.00059, 0.00064, 0.00058, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00057, 0.00059, 0.0006, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00065, 0.00058, 0.00059, 0.00058, 0.00064, 0.00059, 0.00059, 0.00059, 0.00062, 0.00059, 0.00064, 0.00059, 0.00059, 0.00059, 0.0006, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00064, 0.00065, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00061, 0.0006, 0.00067, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00057, 0.00059, 0.00059, 0.00061, 0.00059, 0.0006, 0.00064, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.0006, 0.00058, 0.00059, 0.0006, 0.00059, 0.00059, 0.00057, 0.00058, 0.00058, 0.00058, 0.0006, 0.0006, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00064, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00062, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00063, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00064, 0.0006, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.0006, 0.00064, 0.00058, 0.00058, 0.0006, 0.0006, 0.00057, 0.00058, 0.00059, 0.00059, 0.00059, 0.00062, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.0006, 0.00059, 0.00059, 0.00058, 0.00058, 0.00064, 0.00059, 0.00064, 0.00058, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00065, 0.0006, 0.00057, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.0006, 0.00058, 0.00058, 0.00059, 0.00059, 0.00057, 0.00058, 0.00057, 0.00064, 0.00057, 0.00058, 0.00068, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00057, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00057, 0.00059, 0.00062, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.0006, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00058, 0.00057, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00057, 0.00059, 0.00058, 0.00059, 0.00059, 0.0006, 0.00058, 0.00058, 0.00059, 0.00058, 0.00071, 0.00058, 0.00064, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00063, 0.00059, 0.00058, 0.00058, 0.00057, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00065, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00057, 0.00058, 0.00058, 0.00059, 0.00059, 0.00069, 0.00058, 0.0006, 0.00058, 0.00058, 0.00057, 0.00058, 0.00057, 0.00059, 0.00058, 0.00058]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00021, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00016, 0.00014, 0.00014, 0.00014, 0.0002, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.22691, 0.00055, 0.00056, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00055, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00057, 0.00057, 0.00056, 0.00056, 0.00054, 0.00056, 0.00056, 0.00055, 0.00055, 0.00056, 0.00056, 0.00055, 0.00061, 0.00058, 0.00058, 0.00056, 0.00056, 0.00056, 0.00057, 0.00061, 0.00059, 0.00057, 0.00058, 0.00056, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00056, 0.00058, 0.00058, 0.00059, 0.00057, 0.00059, 0.00057, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.0006, 0.00057, 0.00058, 0.00058, 0.00056, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00057, 0.0006, 0.00061, 0.00058, 0.00059, 0.00058, 0.00057, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00056, 0.00057, 0.00058, 0.00059, 0.00058, 0.00057, 0.00057, 0.00058, 0.00057, 0.00058, 0.00058, 0.00056, 0.00057, 0.00049, 0.00057, 0.00057, 0.00057, 0.00048, 0.00057, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00048, 0.00048, 0.0005, 0.00058, 0.0006, 0.00058, 0.00058, 0.00059, 0.00056, 0.00058, 0.00058, 0.00058, 0.00059, 0.00057, 0.00058, 0.00057, 0.00058, 0.00057, 0.00073, 0.00058, 0.00058, 0.00057, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00046, 0.00058, 0.00057, 0.00059, 0.00058, 0.00057, 0.00048, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00057, 0.00059, 0.00058, 0.00057, 0.00057, 0.00058, 0.00056, 0.00058, 0.00058, 0.00058, 0.00057, 0.00047, 0.00047, 0.00067, 0.00057, 0.00058, 0.00059, 0.00057, 0.00058, 0.00066, 0.00058, 0.00058, 0.00059, 0.00048, 0.00059, 0.00059, 0.00059, 0.00057, 0.00062, 0.00058, 0.00057, 0.00057, 0.00057, 0.00058, 0.0006, 0.00057, 0.00057, 0.00058, 0.00058, 0.00057, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.0006, 0.00058, 0.00058, 0.00058, 0.00064, 0.00057, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00057, 0.00057, 0.0006, 0.00058, 0.00057, 0.00058, 0.00059, 0.00059, 0.0006, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.0006, 0.00058, 0.00061, 0.00059, 0.00057, 0.00056, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00063, 0.0006, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00061, 0.00059, 0.0006, 0.00058, 0.0006, 0.0006, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00057, 0.0006, 0.0006, 0.00059, 0.00059, 0.00059, 0.0006, 0.00059, 0.0006, 0.00059, 0.00058, 0.00058, 0.00057, 0.00058, 0.00061, 0.00058, 0.00061, 0.00058, 0.00058, 0.00057, 0.00057, 0.00059, 0.00058, 0.00057, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.0006, 0.00058, 0.0006, 0.00057, 0.0006, 0.00059, 0.00059, 0.00059, 0.0006, 0.0006, 0.00059, 0.00058, 0.0006, 0.00058, 0.0006, 0.0006, 0.00061, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00061, 0.00062, 0.00062, 0.00058, 0.00057, 0.00058, 0.0006, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00063, 0.0006, 0.00059, 0.00062, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00063, 0.00059, 0.00056, 0.00058, 0.00058, 0.00056, 0.00057, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.0006, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.0006, 0.00058, 0.0006, 0.00058, 0.00059, 0.00058, 0.00057, 0.00057, 0.0006, 0.00064, 0.00059, 0.00061, 0.00058, 0.00058, 0.0006, 0.00058, 0.0006, 0.00067, 0.00057, 0.00058, 0.0006, 0.00059]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00354, 0.00262, 0.00261, 0.00266, 0.0026, 0.0026, 0.0026, 0.00261, 0.00259, 0.00259, 0.00261, 0.00261, 0.00261, 0.00262, 0.00262, 0.0026, 0.0026, 0.00258, 0.00264, 0.00259, 0.00269, 0.00267, 0.00262, 0.00291, 0.00262, 0.00271, 0.00259, 0.00259, 0.0026, 0.00261, 0.00261, 0.0026, 0.0026, 0.00257, 0.00262, 0.00261, 0.00262, 0.00265, 0.0026, 0.00261, 0.00261, 0.00259, 0.0026, 0.00265, 0.00262, 0.00261, 0.00265, 0.00258, 0.0026, 0.00263, 0.00261, 0.0026, 0.0026, 0.00258, 0.00258, 0.0026, 0.00261, 0.0026, 0.00261, 0.00261, 0.00263, 0.00259, 0.00262, 0.0026, 0.00261, 0.00258, 0.00261, 0.0026, 0.00267, 0.00261, 0.00258, 0.00265, 0.00259, 0.00261, 0.00258, 0.00258, 0.00261, 0.00261, 0.00261, 0.00259, 0.00258, 0.00262, 0.00261, 0.00261, 0.00261, 0.00259, 0.00262, 0.0026, 0.0026, 0.00259, 0.0026, 0.00261, 0.0026, 0.00261, 0.0026, 0.00272, 0.00259, 0.00262, 0.00257, 0.0026, 0.00261, 0.00259, 0.00263, 0.00259, 0.00261, 0.00261, 0.00267, 0.00258, 0.0026, 0.00259, 0.00262, 0.00259, 0.00259, 0.00481, 0.00261, 0.00259, 0.00263, 0.0029, 0.00259, 0.00261, 0.00263, 0.0026, 0.0026, 0.00261, 0.00261, 0.00262, 0.00261, 0.00259, 0.0026, 0.00308, 0.00357, 0.00364, 0.0026, 0.00259, 0.00266, 0.00258, 0.0026, 0.00264, 0.00261, 0.0026, 0.0026, 0.0026, 0.00261, 0.00261, 0.0026, 0.00258, 0.00262, 0.00262, 0.00264, 0.00258, 0.00262, 0.0026, 0.00259, 0.00268, 0.0026, 0.00263, 0.00257, 0.0026, 0.00259, 0.00262, 0.00262, 0.00261, 0.00261, 0.00261, 0.0026, 0.0026, 0.00261, 0.0026, 0.00266, 0.00266, 0.00264, 0.0027, 0.00268, 0.00266, 0.00266, 0.00267, 0.00263, 0.00266, 0.00264, 0.00459, 0.00266, 0.00266, 0.00267, 0.00266, 0.00265, 0.00269, 0.00266, 0.00267, 0.00272, 0.00267, 0.00265, 0.00272, 0.00266, 0.00266, 0.0027, 0.00266, 0.00265, 0.00269, 0.00265, 0.00265, 0.00265, 0.00268, 0.00265, 0.00266, 0.00266, 0.00267, 0.00266, 0.00265, 0.00267, 0.00266, 0.0027, 0.00266, 0.00264, 0.00266, 0.00264, 0.00266, 0.00265, 0.00265, 0.00266, 0.00268, 0.00268, 0.00266, 0.00266, 0.00266, 0.00264, 0.00265, 0.00269, 0.00267, 0.00267, 0.00269, 0.00266, 0.00266, 0.00266, 0.00266, 0.00265, 0.00268, 0.0027, 0.00351, 0.00265, 0.00266, 0.00267, 0.00267, 0.00265, 0.00267, 0.00265, 0.00267, 0.00266, 0.00266, 0.00275, 0.00266, 0.00264, 0.00265, 0.00266, 0.0027, 0.00287, 0.00267, 0.00306, 0.00267, 0.00265, 0.00268, 0.00266, 0.00266, 0.00265, 0.00265, 0.00265, 0.00266, 0.00271, 0.00266, 0.00266, 0.00267, 0.00267, 0.00273, 0.00267, 0.00267, 0.00264, 0.00267, 0.00266, 0.00264, 0.00267, 0.00267, 0.00266, 0.00267, 0.00266, 0.00263, 0.00266, 0.00268, 0.00265, 0.00266, 0.00266, 0.00267, 0.00267, 0.00265, 0.00268, 0.00266, 0.00267, 0.00272, 0.00264, 0.00266, 0.00266, 0.00265, 0.00277, 0.00266, 0.00269, 0.00264, 0.00265, 0.00266, 0.00259, 0.00259, 0.0026, 0.00261, 0.0026, 0.00262, 0.0026, 0.00261, 0.00261, 0.00261, 0.00261, 0.00272, 0.00262, 0.00323, 0.0026, 0.00261, 0.00262, 0.00269, 0.00259, 0.00261, 0.00261, 0.00261, 0.00261, 0.0026, 0.00259, 0.00258, 0.0026, 0.00262, 0.00261, 0.00261, 0.00262, 0.0026, 0.0026, 0.00264, 0.00259, 0.00285, 0.0026, 0.00259, 0.00259, 0.0026, 0.00258, 0.00261, 0.00261, 0.00259, 0.0026, 0.00261, 0.0026, 0.00273, 0.0026, 0.00258, 0.00261, 0.0026, 0.00259, 0.0026, 0.00259, 0.00259, 0.00261, 0.00266, 0.00266, 0.00265, 0.00269, 0.00269, 0.00266, 0.00266, 0.00266, 0.00264, 0.00266, 0.00267, 0.00265, 0.00273, 0.00265, 0.00265, 0.0027, 0.00266, 0.00274, 0.00267, 0.00267, 0.00267, 0.00266, 0.00266, 0.00266, 0.00299, 0.00266, 0.00268, 0.00265, 0.00267, 0.00265, 0.00268, 0.00265, 0.00266, 0.00267, 0.00267, 0.00271, 0.00267]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00249, 0.00044, 0.00044, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00044, 0.00044, 0.00044, 0.00044, 0.00045, 0.00044, 0.00044, 0.00045, 0.00044, 0.00044, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00047, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00048, 0.00056, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00047, 0.00046, 0.00049, 0.00051, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00045, 0.00047, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00049, 0.00048, 0.00046, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00047, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00048, 0.00046, 0.00046, 0.00047, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.0005, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00045, 0.00046, 0.00047, 0.00045, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00057, 0.00046, 0.00046, 0.00047, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00044, 0.00046, 0.00046, 0.00047, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00046, 0.00046]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00056, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00048, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00048, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00048, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00069, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00053, 0.00064, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00052, 0.00049, 0.00049, 0.00051, 0.00049, 0.0005, 0.00051, 0.00049, 0.00049, 0.00053, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00051, 0.00049, 0.00049, 0.00059, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00068, 0.0005, 0.00049, 0.00049, 0.00049, 0.00077, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00062, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.0005, 0.00064, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00061, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00052, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.23567, 0.00458, 0.00457, 0.00463, 0.00456, 0.00458, 0.00456, 0.00457, 0.00457, 0.00456, 0.00457, 0.00457, 0.00457, 0.00456, 0.00459, 0.00457, 0.00455, 0.00458, 0.00456, 0.00456, 0.00465, 0.00463, 0.00457, 0.005, 0.00457, 0.00468, 0.0046, 0.00458, 0.00461, 0.0046, 0.00456, 0.00456, 0.00462, 0.00463, 0.00464, 0.0046, 0.00464, 0.00464, 0.00461, 0.00462, 0.00462, 0.00459, 0.00465, 0.00464, 0.00462, 0.00462, 0.00467, 0.00457, 0.00462, 0.00465, 0.00462, 0.00462, 0.00473, 0.00459, 0.0046, 0.00464, 0.00463, 0.00458, 0.00462, 0.00462, 0.00462, 0.00459, 0.00465, 0.00461, 0.00463, 0.00459, 0.0046, 0.00462, 0.00469, 0.00466, 0.00461, 0.00468, 0.0046, 0.00461, 0.0046, 0.00464, 0.00463, 0.00465, 0.00465, 0.00462, 0.00459, 0.00459, 0.00461, 0.00461, 0.00462, 0.00461, 0.00463, 0.00459, 0.00461, 0.00458, 0.00461, 0.00463, 0.00459, 0.0046, 0.00456, 0.00476, 0.00459, 0.00465, 0.00449, 0.00462, 0.00463, 0.0046, 0.00465, 0.0046, 0.00462, 0.00462, 0.00468, 0.00461, 0.00462, 0.00462, 0.00464, 0.0045, 0.00453, 0.00715, 0.00463, 0.00463, 0.00466, 0.00492, 0.00461, 0.00459, 0.00464, 0.00466, 0.00461, 0.00462, 0.00461, 0.00464, 0.00462, 0.00461, 0.0046, 0.00561, 0.00589, 0.00578, 0.0046, 0.0046, 0.00467, 0.0046, 0.00462, 0.00468, 0.00449, 0.00462, 0.00461, 0.00464, 0.00463, 0.00464, 0.0045, 0.0046, 0.00464, 0.00464, 0.00466, 0.00463, 0.00464, 0.00464, 0.00462, 0.00469, 0.00461, 0.00467, 0.00459, 0.00458, 0.00465, 0.00466, 0.00462, 0.00464, 0.00454, 0.00452, 0.00487, 0.00461, 0.00461, 0.00463, 0.00466, 0.00467, 0.00477, 0.00473, 0.00469, 0.00473, 0.00459, 0.00473, 0.00467, 0.00467, 0.00466, 0.0068, 0.00467, 0.00466, 0.00467, 0.00465, 0.00466, 0.00472, 0.00467, 0.00466, 0.00474, 0.00468, 0.00464, 0.00474, 0.00468, 0.00473, 0.00472, 0.00468, 0.0047, 0.00472, 0.00465, 0.00466, 0.00496, 0.00468, 0.00467, 0.00471, 0.0047, 0.00468, 0.00472, 0.00467, 0.00467, 0.00466, 0.00472, 0.00469, 0.00466, 0.00464, 0.00467, 0.00469, 0.00466, 0.00468, 0.00469, 0.00474, 0.00473, 0.00468, 0.0047, 0.00468, 0.00467, 0.00469, 0.00477, 0.00469, 0.00464, 0.00465, 0.0047, 0.0047, 0.00469, 0.00468, 0.00472, 0.00469, 0.00472, 0.00563, 0.00469, 0.00469, 0.00469, 0.0047, 0.00467, 0.0047, 0.00467, 0.00467, 0.00472, 0.00469, 0.00478, 0.00471, 0.00475, 0.00469, 0.00469, 0.00472, 0.00495, 0.00468, 0.0051, 0.00473, 0.0047, 0.00468, 0.00485, 0.00471, 0.00466, 0.0047, 0.00468, 0.00471, 0.00473, 0.00471, 0.0047, 0.00469, 0.00469, 0.00472, 0.00468, 0.00471, 0.00464, 0.00469, 0.00465, 0.00469, 0.00468, 0.00465, 0.00471, 0.00469, 0.0047, 0.00498, 0.00469, 0.00468, 0.00467, 0.00468, 0.00506, 0.0047, 0.00468, 0.00467, 0.00466, 0.00468, 0.0047, 0.00474, 0.00468, 0.00469, 0.0047, 0.00467, 0.00478, 0.00468, 0.00471, 0.0047, 0.00469, 0.00471, 0.00461, 0.00466, 0.00461, 0.00462, 0.0046, 0.00465, 0.00463, 0.00465, 0.00465, 0.00468, 0.00461, 0.00471, 0.00465, 0.00542, 0.00464, 0.00463, 0.00463, 0.00472, 0.0046, 0.00464, 0.00463, 0.0048, 0.00465, 0.00463, 0.00461, 0.00463, 0.0046, 0.00463, 0.00465, 0.00464, 0.00463, 0.00463, 0.00465, 0.00469, 0.00459, 0.00495, 0.00468, 0.00461, 0.00465, 0.00461, 0.00464, 0.00464, 0.00466, 0.00462, 0.00464, 0.00508, 0.00461, 0.0048, 0.00463, 0.00454, 0.00463, 0.00461, 0.00456, 0.0046, 0.00466, 0.00462, 0.00465, 0.00468, 0.00486, 0.00469, 0.00471, 0.00469, 0.00468, 0.00468, 0.00467, 0.00468, 0.00468, 0.00471, 0.00469, 0.00474, 0.00469, 0.00467, 0.00472, 0.00467, 0.00477, 0.00472, 0.00471, 0.00468, 0.00467, 0.00465, 0.00469, 0.00513, 0.00471, 0.00489, 0.00466, 0.00469, 0.00468, 0.00474, 0.00467, 0.00475, 0.00467, 0.00469, 0.00476, 0.0047]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84424, 10.87342, 10.85055, 10.81078, 10.64469, 10.6386, 10.4283, 10.13518, 9.93546, 9.83538, 9.5857, 9.84804, 9.88588, 9.63127, 9.79022, 9.5114, 9.4597, 9.65546, 9.38988, 9.33928, 9.24947, 9.15126, 9.18199, 9.00445, 9.19836, 9.06663, 9.16101, 9.1698, 9.30057, 8.98927, 8.92967, 9.05035, 9.04657, 8.66029, 8.72527, 8.75664, 8.69468, 8.74328, 8.66681, 8.77286, 8.67044, 8.86119, 8.84295, 8.50873, 8.39852, 8.43801, 8.49532, 8.39321, 8.44017, 8.59221, 8.37564, 8.19958, 8.2329, 8.22974, 8.27495, 7.92044, 8.0993, 7.89755, 8.2517, 8.23397, 8.00952, 7.97507, 7.92567, 7.74377, 7.74735, 7.64935, 7.51967, 7.91031, 7.70174, 7.45536, 7.74632, 7.77446, 7.54372, 7.30243, 7.45569, 7.34305, 7.4658, 7.22841, 7.63683, 7.28242, 7.34884, 7.21343, 7.21124, 7.41956, 7.17365, 7.2819, 6.99462, 7.00325, 7.04012, 7.13712, 6.82214, 6.98588, 7.08949, 6.99872, 6.87479, 6.75655, 6.99059, 7.06011, 6.70413, 6.58421, 6.72746, 6.74527, 6.73409, 6.73823, 6.65852, 6.40615, 6.63686, 6.6194, 6.44648, 6.62844, 6.74357, 6.61132, 6.72657, 6.69405, 6.62733, 6.50769, 6.59795, 6.40666, 6.66519, 6.24881, 6.25106, 6.30401, 6.39198, 6.34989, 6.45173, 6.29422, 6.33969, 6.23719, 6.20153, 6.39655, 6.32455, 6.32086, 6.16315, 6.15667, 6.23617, 6.38123, 6.19858, 6.14609, 6.17459, 6.11003, 6.05359, 6.06531, 6.24848, 6.39923, 6.24762, 6.28436, 6.08885, 6.1659, 5.99117, 6.01964, 5.94446, 6.23937, 6.17942, 5.95871, 5.7764, 6.11339, 5.84425, 6.10156, 5.77953, 6.15415, 6.13822, 6.07746, 5.92004, 6.10968, 5.93741, 6.19122, 5.88685, 5.78306, 5.77148, 5.68041, 6.00813, 5.99187, 6.05986, 5.88016, 6.03137, 5.96131, 5.99374, 5.98716, 5.94573, 5.83722, 5.94198, 5.61328, 5.69729, 5.88553, 5.83625, 5.85543, 5.75718, 5.83246, 5.71985, 5.55522, 5.71497, 5.61505, 5.82338, 5.59492, 5.70181, 5.69956, 5.89291, 5.6334, 5.84186, 5.73328, 5.86061, 5.32413, 5.89063, 5.86923, 5.84806, 5.40969, 5.40238, 5.62094, 5.5916, 5.47979, 5.57337, 5.67122, 5.47407, 5.73944, 5.51167, 5.59101, 5.62347, 5.61736, 5.50921, 5.61182, 5.67274, 5.68001, 5.58479, 5.65971, 5.37206, 5.67757, 5.62674, 5.42131, 5.58249, 5.62904, 5.55375, 5.34106, 5.53431, 5.48176, 5.48104, 5.38026, 5.55107, 5.59981, 5.38504, 5.51817, 5.48713, 5.33135, 5.50212, 5.40894, 5.44244, 5.31335, 5.06368, 5.47625, 5.56822, 5.71202, 5.40926, 5.59783, 5.63205, 5.23113, 5.2684, 5.39256, 5.39509, 5.32651, 5.49543, 5.18174, 5.2944, 5.24351, 5.3743, 5.25187, 5.4403, 5.53394, 5.30526, 5.42762, 5.33573, 5.07536, 5.30828, 5.24915, 5.30097, 5.10794, 5.27462, 5.25882, 5.46931, 5.15605, 5.26147, 5.20567, 5.34991, 4.9789, 4.90972, 5.32269, 5.39016, 5.22419, 5.31593, 5.10145, 5.16054, 5.25953, 5.0667, 5.26007, 5.06659, 5.33924, 5.2437, 5.14669, 5.24181, 5.03908, 5.31189, 5.0508, 5.02718, 5.13824, 5.11134, 5.26999, 5.14813, 5.27491, 5.09204, 5.0944, 5.24441, 5.32532, 5.25266, 5.18964, 5.14218, 5.28959, 4.95048, 5.2045, 5.09444, 5.30302, 5.17003, 5.18518, 5.11668, 4.98204, 4.99495, 5.222, 5.30847, 5.098, 5.05553, 4.91636, 5.12137, 5.11611, 4.9291, 5.33462, 5.02406, 5.09871, 5.16424, 5.00257, 5.06588, 5.06465, 4.99336, 5.07822, 5.15996, 4.97519, 5.18105, 4.9261, 4.91748, 5.06072, 4.99116, 4.90494, 4.77574, 4.94081, 5.11232, 5.01149, 5.01672, 5.32706, 4.95549, 4.99178, 5.04351, 4.80691, 4.73281, 4.99471, 5.04386, 4.87342, 4.9541, 5.04639, 5.02142, 4.81154, 4.89155, 4.90243, 4.82954, 4.73696, 5.00591, 4.75497, 5.20346, 4.791, 4.99509, 4.73426, 4.7815, 4.81632, 4.64705, 4.65335, 4.84192, 4.80637, 4.79718, 4.91906, 4.87982, 4.9259, 4.76993, 4.87999, 4.73114, 4.91345, 4.95513, 4.87047, 4.70341, 4.77964, 4.89818, 4.70591, 4.85482, 4.68983, 4.68887, 4.64189]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84424, 10.87342, 10.85055, 10.81078, 10.64469, 10.6386, 10.4283, 10.13518, 9.93546, 9.83538, 9.5857, 9.84804, 9.88588, 9.63127, 9.79022, 9.5114, 9.4597, 9.65546, 9.38988, 9.33928, 9.24947, 9.15126, 9.18199, 9.00445, 9.19836, 9.06663, 9.16101, 9.1698, 9.30057, 8.98927, 8.92967, 9.05035, 9.04657, 8.66029, 8.72527, 8.75664, 8.69468, 8.74328, 8.66681, 8.77286, 8.67044, 8.86119, 8.84295, 8.50873, 8.39852, 8.43801, 8.49532, 8.39321, 8.44017, 8.59221, 8.37564, 8.19958, 8.2329, 8.22974, 8.27495, 7.92044, 8.0993, 7.89755, 8.2517, 8.23397, 8.00952, 7.97507, 7.92567, 7.74377, 7.74735, 7.64935, 7.51967, 7.91031, 7.70174, 7.45536, 7.74632, 7.77446, 7.54372, 7.30243, 7.45569, 7.34305, 7.4658, 7.22841, 7.63683, 7.28242, 7.34884, 7.21343, 7.21124, 7.41956, 7.17365, 7.2819, 6.99462, 7.00325, 7.04012, 7.13712, 6.82214, 6.98588, 7.08949, 6.99872, 6.87479, 6.75655, 6.99059, 7.06011, 6.70413, 6.58421, 6.72746, 6.74527, 6.73409, 6.73823, 6.65852, 6.40615, 6.63686, 6.6194, 6.44648, 6.62844, 6.74357, 6.61132, 6.72657, 6.69405, 6.62733, 6.50769, 6.59795, 6.40666, 6.66519, 6.24881, 6.25106, 6.30401, 6.39198, 6.34989, 6.45173, 6.29422, 6.33969, 6.23719, 6.20153, 6.39655, 6.32455, 6.32086, 6.16315, 6.15667, 6.23617, 6.38123, 6.19858, 6.14609, 6.17459, 6.11003, 6.05359, 6.06531, 6.24848, 6.39923, 6.24762, 6.28436, 6.08885, 6.1659, 5.99117, 6.01964, 5.94446, 6.23937, 6.17942, 5.95871, 5.7764, 6.11339, 5.84425, 6.10156, 5.77953, 6.15415, 6.13822, 6.07746, 5.92004, 6.10968, 5.93741, 6.19122, 5.88685, 5.78306, 5.77148, 5.68041, 6.00813, 5.99187, 6.05986, 5.88016, 6.03137, 5.96131, 5.99374, 5.98716, 5.94573, 5.83722, 5.94198, 5.61328, 5.69729, 5.88553, 5.83625, 5.85543, 5.75718, 5.83246, 5.71985, 5.55522, 5.71497, 5.61505, 5.82338, 5.59492, 5.70181, 5.69956, 5.89291, 5.6334, 5.84186, 5.73328, 5.86061, 5.32413, 5.89063, 5.86923, 5.84806, 5.40969, 5.40238, 5.62094, 5.5916, 5.47979, 5.57337, 5.67122, 5.47407, 5.73944, 5.51167, 5.59101, 5.62347, 5.61736, 5.50921, 5.61182, 5.67274, 5.68001, 5.58479, 5.65971, 5.37206, 5.67757, 5.62674, 5.42131, 5.58249, 5.62904, 5.55375, 5.34106, 5.53431, 5.48176, 5.48104, 5.38026, 5.55107, 5.59981, 5.38504, 5.51817, 5.48713, 5.33135, 5.50212, 5.40894, 5.44244, 5.31335, 5.06368, 5.47625, 5.56822, 5.71202, 5.40926, 5.59783, 5.63205, 5.23113, 5.2684, 5.39256, 5.39509, 5.32651, 5.49543, 5.18174, 5.2944, 5.24351, 5.3743, 5.25187, 5.4403, 5.53394, 5.30526, 5.42762, 5.33573, 5.07536, 5.30828, 5.24915, 5.30097, 5.10794, 5.27462, 5.25882, 5.46931, 5.15605, 5.26147, 5.20567, 5.34991, 4.9789, 4.90972, 5.32269, 5.39016, 5.22419, 5.31593, 5.10145, 5.16054, 5.25953, 5.0667, 5.26007, 5.06659, 5.33924, 5.2437, 5.14669, 5.24181, 5.03908, 5.31189, 5.0508, 5.02718, 5.13824, 5.11134, 5.26999, 5.14813, 5.27491, 5.09204, 5.0944, 5.24441, 5.32532, 5.25266, 5.18964, 5.14218, 5.28959, 4.95048, 5.2045, 5.09444, 5.30302, 5.17003, 5.18518, 5.11668, 4.98204, 4.99495, 5.222, 5.30847, 5.098, 5.05553, 4.91636, 5.12137, 5.11611, 4.9291, 5.33462, 5.02406, 5.09871, 5.16424, 5.00257, 5.06588, 5.06465, 4.99336, 5.07822, 5.15996, 4.97519, 5.18105, 4.9261, 4.91748, 5.06072, 4.99116, 4.90494, 4.77574, 4.94081, 5.11232, 5.01149, 5.01672, 5.32706, 4.95549, 4.99178, 5.04351, 4.80691, 4.73281, 4.99471, 5.04386, 4.87342, 4.9541, 5.04639, 5.02142, 4.81154, 4.89155, 4.90243, 4.82954, 4.73696, 5.00591, 4.75497, 5.20346, 4.791, 4.99509, 4.73426, 4.7815, 4.81632, 4.64705, 4.65335, 4.84192, 4.80637, 4.79718, 4.91906, 4.87982, 4.9259, 4.76993, 4.87999, 4.73114, 4.91345, 4.95513, 4.87047, 4.70341, 4.77964, 4.89818, 4.70591, 4.85482, 4.68983, 4.68887, 4.64189]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.93626, 13.32689, 13.8137, 12.62172, 11.96992, 9.43513, 6.80799, 6.88665, 5.95498, 4.54619, 4.13053, 2.82596, 2.39543, 2.34537, 2.05773, 2.21996, 2.14537, 1.88392, 2.17069, 2.06105, 2.12373, 2.16615, 2.00976, 2.20876, 1.97308, 2.09194, 1.90863, 1.88776, 1.95054, 2.15308, 2.08778, 2.10616, 1.95646, 2.17094, 2.31724, 2.02642, 2.04764, 1.84545, 1.93704, 1.75657, 2.13069, 1.75993, 1.70876, 1.86665, 1.92331, 1.79127, 1.74297, 1.74426, 1.75161, 1.53485, 1.75292, 1.73299, 1.79809, 1.83477, 1.59059, 1.79085, 1.74313, 1.81505, 1.54888, 1.47615, 1.68285, 1.4812, 1.79315, 1.92171, 1.63149, 1.63813, 1.6586, 1.59744, 1.47545, 1.65909, 1.42464, 1.41939, 1.49901, 1.42049, 1.40172, 1.46225, 1.44185, 1.3706, 1.36838, 1.26055, 1.34627, 1.29904, 1.25687, 1.20642, 1.27731, 1.27576, 1.4537, 1.34738, 1.41703, 1.10279, 1.09805, 1.25584, 1.13228, 1.20775, 0.93229, 1.32305, 1.10083, 1.31134, 0.99675, 1.32116, 1.31807, 1.20377, 1.14298, 1.25982, 1.11587, 1.06268, 1.1383, 1.13456, 1.18344, 1.01042, 1.19822, 0.96542, 0.98282, 0.98083, 1.21915, 1.08304, 1.00478, 1.26788, 1.10619, 1.30807, 1.1248, 1.36119, 1.37901, 1.4392, 1.56444, 1.29037, 1.19911, 1.00927, 1.14759, 1.2293, 1.07062, 1.374, 1.0323, 1.06393, 1.18259, 1.20195, 1.16586, 1.44753, 0.94529, 1.13538, 1.05269, 1.34467, 1.18959, 1.01819, 0.86119, 1.06946, 1.34129, 1.684, 1.13519, 1.32985, 1.38775, 1.34761, 1.74434, 1.43622, 1.39335, 1.37538, 1.86703, 2.00418, 1.35288, 1.23486, 1.3698, 1.32764, 0.9773, 0.96112, 1.19304, 1.38421, 1.30281, 1.24815, 1.29487, 1.60508, 1.50397, 1.88527, 1.44501, 1.35752, 0.94887, 1.377, 2.16776, 1.36769, 1.5918, 1.53974, 1.46219, 1.57752, 1.18503, 1.28159, 1.42022, 1.06676, 1.57312, 1.38623, 1.21566, 1.67634, 1.0445, 1.27733, 1.33704, 1.42129, 1.46397, 1.28187, 1.4299, 1.30773, 1.5098, 1.44392, 1.45291, 1.64364, 1.49176, 1.37459, 1.51541, 1.63213, 1.48678, 1.52484, 1.4594, 1.29967, 1.2736, 1.3991, 1.32876, 1.30752, 2.30271, 1.55904, 1.8449, 1.46033, 1.24296, 1.20709, 1.62628, 1.5864, 1.26763, 1.43759, 1.47487, 1.37697, 1.3542, 1.33151, 1.73529, 1.34567, 1.25198, 1.32539, 1.47482, 1.18237, 1.36743, 1.49708, 1.35135, 1.39444, 1.32979, 1.17935, 1.87393, 1.4264, 1.47427, 1.49289, 1.23046, 1.40513, 1.22641, 1.41026, 1.60243, 1.3143, 1.19178, 1.29275, 1.40778, 1.27321, 1.41008, 1.70248, 1.64394, 1.51805, 1.52213, 1.56958, 1.37322, 1.23197, 1.2534, 1.33391, 1.27155, 1.71409, 1.36328, 1.34111, 1.56216, 1.69178, 1.34859, 1.23125, 1.30141, 1.35618, 1.71086, 1.21378, 1.62762, 1.35769, 1.32471, 1.3449, 1.37393, 1.16861, 1.52125, 1.65464, 1.84529, 1.4419, 1.39298, 1.45439, 1.43606, 1.60436, 1.56537, 1.49466, 1.35372, 1.44924, 1.44717, 1.59557, 1.51747, 1.64905, 1.33058, 1.31553, 1.61355, 1.23394, 1.40751, 1.24118, 1.39003, 1.46524, 1.46231, 1.5848, 1.30142, 1.49751, 1.49494, 1.35146, 1.32779, 1.48392, 1.42067, 1.43745, 1.57573, 1.52413, 1.22763, 1.19418, 1.89055, 1.53347, 1.40105, 1.60967, 1.38946, 1.31243, 1.45306, 1.42686, 1.36629, 1.4597, 1.59178, 1.37262, 1.28569, 1.49855, 1.29513, 1.26508, 1.32564, 1.18627, 1.52963, 1.41157, 1.22284, 1.09058, 1.41662, 1.39267, 1.29437, 1.39958, 1.3399, 1.36221, 1.4319, 1.07457, 1.45594, 1.29022, 1.47328, 1.63456, 1.35731, 1.53342, 1.23853, 1.30778, 1.37885, 1.39437, 1.58806, 1.41021, 1.41084, 1.3741, 1.18704, 1.36438, 1.50507, 1.3615, 1.43368, 1.39267, 1.48306, 1.60864, 1.92464, 1.65072, 1.54144, 1.35616, 1.29657, 1.5044, 1.29558, 1.3191, 1.41541, 1.44176, 1.48919, 1.28271, 1.18322, 1.31948, 1.34975, 1.36515, 1.26883, 1.48957, 1.40195, 1.45318, 1.67399, 1.47474, 1.53573, 1.49973, 1.39375, 1.51272, 1.36339, 1.21633]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.93626, 13.32689, 13.8137, 12.62172, 11.96992, 9.43513, 6.80799, 6.88665, 5.95498, 4.54619, 4.13053, 2.82596, 2.39543, 2.34537, 2.05773, 2.21996, 2.14537, 1.88392, 2.17069, 2.06105, 2.12373, 2.16615, 2.00976, 2.20876, 1.97308, 2.09194, 1.90863, 1.88776, 1.95054, 2.15308, 2.08778, 2.10616, 1.95646, 2.17094, 2.31724, 2.02642, 2.04764, 1.84545, 1.93704, 1.75657, 2.13069, 1.75993, 1.70876, 1.86665, 1.92331, 1.79127, 1.74297, 1.74426, 1.75161, 1.53485, 1.75292, 1.73299, 1.79809, 1.83477, 1.59059, 1.79085, 1.74313, 1.81505, 1.54888, 1.47615, 1.68285, 1.4812, 1.79315, 1.92171, 1.63149, 1.63813, 1.6586, 1.59744, 1.47545, 1.65909, 1.42464, 1.41939, 1.49901, 1.42049, 1.40172, 1.46225, 1.44185, 1.3706, 1.36838, 1.26055, 1.34627, 1.29904, 1.25687, 1.20642, 1.27731, 1.27576, 1.4537, 1.34738, 1.41703, 1.10279, 1.09805, 1.25584, 1.13228, 1.20775, 0.93229, 1.32305, 1.10083, 1.31134, 0.99675, 1.32116, 1.31807, 1.20377, 1.14298, 1.25982, 1.11587, 1.06268, 1.1383, 1.13456, 1.18344, 1.01042, 1.19822, 0.96542, 0.98282, 0.98083, 1.21915, 1.08304, 1.00478, 1.26788, 1.10619, 1.30807, 1.1248, 1.36119, 1.37901, 1.4392, 1.56444, 1.29037, 1.19911, 1.00927, 1.14759, 1.2293, 1.07062, 1.374, 1.0323, 1.06393, 1.18259, 1.20195, 1.16586, 1.44753, 0.94529, 1.13538, 1.05269, 1.34467, 1.18959, 1.01819, 0.86119, 1.06946, 1.34129, 1.684, 1.13519, 1.32985, 1.38775, 1.34761, 1.74434, 1.43622, 1.39335, 1.37538, 1.86703, 2.00418, 1.35288, 1.23486, 1.3698, 1.32764, 0.9773, 0.96112, 1.19304, 1.38421, 1.30281, 1.24815, 1.29487, 1.60508, 1.50397, 1.88527, 1.44501, 1.35752, 0.94887, 1.377, 2.16776, 1.36769, 1.5918, 1.53974, 1.46219, 1.57752, 1.18503, 1.28159, 1.42022, 1.06676, 1.57312, 1.38623, 1.21566, 1.67634, 1.0445, 1.27733, 1.33704, 1.42129, 1.46397, 1.28187, 1.4299, 1.30773, 1.5098, 1.44392, 1.45291, 1.64364, 1.49176, 1.37459, 1.51541, 1.63213, 1.48678, 1.52484, 1.4594, 1.29967, 1.2736, 1.3991, 1.32876, 1.30752, 2.30271, 1.55904, 1.8449, 1.46033, 1.24296, 1.20709, 1.62628, 1.5864, 1.26763, 1.43759, 1.47487, 1.37697, 1.3542, 1.33151, 1.73529, 1.34567, 1.25198, 1.32539, 1.47482, 1.18237, 1.36743, 1.49708, 1.35135, 1.39444, 1.32979, 1.17935, 1.87393, 1.4264, 1.47427, 1.49289, 1.23046, 1.40513, 1.22641, 1.41026, 1.60243, 1.3143, 1.19178, 1.29275, 1.40778, 1.27321, 1.41008, 1.70248, 1.64394, 1.51805, 1.52213, 1.56958, 1.37322, 1.23197, 1.2534, 1.33391, 1.27155, 1.71409, 1.36328, 1.34111, 1.56216, 1.69178, 1.34859, 1.23125, 1.30141, 1.35618, 1.71086, 1.21378, 1.62762, 1.35769, 1.32471, 1.3449, 1.37393, 1.16861, 1.52125, 1.65464, 1.84529, 1.4419, 1.39298, 1.45439, 1.43606, 1.60436, 1.56537, 1.49466, 1.35372, 1.44924, 1.44717, 1.59557, 1.51747, 1.64905, 1.33058, 1.31553, 1.61355, 1.23394, 1.40751, 1.24118, 1.39003, 1.46524, 1.46231, 1.5848, 1.30142, 1.49751, 1.49494, 1.35146, 1.32779, 1.48392, 1.42067, 1.43745, 1.57573, 1.52413, 1.22763, 1.19418, 1.89055, 1.53347, 1.40105, 1.60967, 1.38946, 1.31243, 1.45306, 1.42686, 1.36629, 1.4597, 1.59178, 1.37262, 1.28569, 1.49855, 1.29513, 1.26508, 1.32564, 1.18627, 1.52963, 1.41157, 1.22284, 1.09058, 1.41662, 1.39267, 1.29437, 1.39958, 1.3399, 1.36221, 1.4319, 1.07457, 1.45594, 1.29022, 1.47328, 1.63456, 1.35731, 1.53342, 1.23853, 1.30778, 1.37885, 1.39437, 1.58806, 1.41021, 1.41084, 1.3741, 1.18704, 1.36438, 1.50507, 1.3615, 1.43368, 1.39267, 1.48306, 1.60864, 1.92464, 1.65072, 1.54144, 1.35616, 1.29657, 1.5044, 1.29558, 1.3191, 1.41541, 1.44176, 1.48919, 1.28271, 1.18322, 1.31948, 1.34975, 1.36515, 1.26883, 1.48957, 1.40195, 1.45318, 1.67399, 1.47474, 1.53573, 1.49973, 1.39375, 1.51272, 1.36339, 1.21633]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [69.0, 86.0, 77.0, 73.0, 78.0, 81.0, 100.0, 105.0, 134.0, 134.0, 122.0, 173.0, 158.0, 179.0, 178.0, 172.0, 173.0, 192.0, 186.0, 185.0, 155.0, 157.0, 183.0, 172.0, 179.0, 162.0, 166.0, 176.0, 162.0, 177.0, 178.0, 149.0, 163.0, 200.0, 122.0, 151.0, 160.0, 216.0, 173.0, 192.0, 163.0, 174.0, 167.0, 195.0, 177.0, 181.0, 195.0, 201.0, 171.0, 240.0, 190.0, 187.0, 177.0, 159.0, 167.0, 211.0, 151.0, 167.0, 226.0, 215.0, 184.0, 206.0, 174.0, 166.0, 203.0, 236.0, 215.0, 192.0, 197.0, 197.0, 250.0, 225.0, 178.0, 210.0, 205.0, 223.0, 233.0, 196.0, 258.0, 221.0, 228.0, 237.0, 226.0, 223.0, 188.0, 182.0, 179.0, 198.0, 147.0, 189.0, 211.0, 214.0, 206.0, 216.0, 245.0, 156.0, 216.0, 214.0, 192.0, 170.0, 167.0, 167.0, 171.0, 168.0, 164.0, 141.0, 174.0, 143.0, 140.0, 184.0, 153.0, 162.0, 175.0, 144.0, 145.0, 144.0, 166.0, 110.0, 159.0, 132.0, 128.0, 137.0, 112.0, 132.0, 126.0, 136.0, 128.0, 172.0, 158.0, 131.0, 135.0, 133.0, 133.0, 144.0, 114.0, 123.0, 127.0, 129.0, 121.0, 139.0, 118.0, 107.0, 135.0, 149.0, 155.0, 123.0, 118.0, 109.0, 109.0, 111.0, 101.0, 119.0, 87.0, 118.0, 99.0, 104.0, 99.0, 88.0, 112.0, 112.0, 136.0, 110.0, 122.0, 128.0, 102.0, 105.0, 114.0, 106.0, 103.0, 119.0, 109.0, 83.0, 87.0, 99.0, 136.0, 116.0, 91.0, 112.0, 94.0, 98.0, 128.0, 100.0, 108.0, 115.0, 104.0, 128.0, 109.0, 99.0, 112.0, 96.0, 123.0, 103.0, 109.0, 84.0, 117.0, 105.0, 92.0, 104.0, 83.0, 96.0, 128.0, 71.0, 107.0, 110.0, 99.0, 96.0, 100.0, 100.0, 99.0, 122.0, 94.0, 98.0, 121.0, 118.0, 83.0, 96.0, 99.0, 123.0, 108.0, 107.0, 108.0, 93.0, 89.0, 101.0, 121.0, 121.0, 113.0, 108.0, 83.0, 123.0, 89.0, 105.0, 99.0, 100.0, 108.0, 105.0, 95.0, 112.0, 101.0, 110.0, 93.0, 108.0, 94.0, 120.0, 118.0, 107.0, 98.0, 121.0, 102.0, 97.0, 111.0, 126.0, 102.0, 108.0, 107.0, 108.0, 95.0, 97.0, 96.0, 118.0, 100.0, 111.0, 103.0, 92.0, 100.0, 101.0, 100.0, 103.0, 112.0, 87.0, 86.0, 119.0, 97.0, 101.0, 119.0, 120.0, 124.0, 114.0, 108.0, 105.0, 101.0, 104.0, 103.0, 98.0, 86.0, 101.0, 115.0, 98.0, 90.0, 108.0, 102.0, 102.0, 108.0, 125.0, 109.0, 90.0, 115.0, 94.0, 114.0, 113.0, 98.0, 113.0, 122.0, 101.0, 97.0, 109.0, 106.0, 105.0, 115.0, 95.0, 117.0, 118.0, 95.0, 111.0, 88.0, 121.0, 121.0, 117.0, 138.0, 134.0, 89.0, 99.0, 117.0, 93.0, 106.0, 123.0, 117.0, 107.0, 117.0, 108.0, 86.0, 121.0, 125.0, 105.0, 114.0, 107.0, 129.0, 114.0, 114.0, 107.0, 120.0, 118.0, 101.0, 109.0, 107.0, 124.0, 120.0, 116.0, 103.0, 127.0, 126.0, 90.0, 102.0, 114.0, 111.0, 108.0, 136.0, 107.0, 112.0, 104.0, 113.0, 117.0, 133.0, 104.0, 125.0, 119.0, 111.0, 122.0, 100.0, 118.0, 119.0, 104.0, 85.0, 133.0, 104.0, 119.0, 118.0, 95.0, 117.0, 123.0, 101.0, 132.0, 121.0, 110.0, 116.0, 116.0, 111.0, 91.0, 104.0, 104.0, 115.0, 124.0, 105.0, 104.0, 105.0, 101.0, 99.0, 112.0, 126.0, 139.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [69.0, 86.0, 77.0, 73.0, 78.0, 81.0, 100.0, 105.0, 134.0, 134.0, 122.0, 173.0, 158.0, 179.0, 178.0, 172.0, 173.0, 192.0, 186.0, 185.0, 155.0, 157.0, 183.0, 172.0, 179.0, 162.0, 166.0, 176.0, 162.0, 177.0, 178.0, 149.0, 163.0, 200.0, 122.0, 151.0, 160.0, 216.0, 173.0, 192.0, 163.0, 174.0, 167.0, 195.0, 177.0, 181.0, 195.0, 201.0, 171.0, 240.0, 190.0, 187.0, 177.0, 159.0, 167.0, 211.0, 151.0, 167.0, 226.0, 215.0, 184.0, 206.0, 174.0, 166.0, 203.0, 236.0, 215.0, 192.0, 197.0, 197.0, 250.0, 225.0, 178.0, 210.0, 205.0, 223.0, 233.0, 196.0, 258.0, 221.0, 228.0, 237.0, 226.0, 223.0, 188.0, 182.0, 179.0, 198.0, 147.0, 189.0, 211.0, 214.0, 206.0, 216.0, 245.0, 156.0, 216.0, 214.0, 192.0, 170.0, 167.0, 167.0, 171.0, 168.0, 164.0, 141.0, 174.0, 143.0, 140.0, 184.0, 153.0, 162.0, 175.0, 144.0, 145.0, 144.0, 166.0, 110.0, 159.0, 132.0, 128.0, 137.0, 112.0, 132.0, 126.0, 136.0, 128.0, 172.0, 158.0, 131.0, 135.0, 133.0, 133.0, 144.0, 114.0, 123.0, 127.0, 129.0, 121.0, 139.0, 118.0, 107.0, 135.0, 149.0, 155.0, 123.0, 118.0, 109.0, 109.0, 111.0, 101.0, 119.0, 87.0, 118.0, 99.0, 104.0, 99.0, 88.0, 112.0, 112.0, 136.0, 110.0, 122.0, 128.0, 102.0, 105.0, 114.0, 106.0, 103.0, 119.0, 109.0, 83.0, 87.0, 99.0, 136.0, 116.0, 91.0, 112.0, 94.0, 98.0, 128.0, 100.0, 108.0, 115.0, 104.0, 128.0, 109.0, 99.0, 112.0, 96.0, 123.0, 103.0, 109.0, 84.0, 117.0, 105.0, 92.0, 104.0, 83.0, 96.0, 128.0, 71.0, 107.0, 110.0, 99.0, 96.0, 100.0, 100.0, 99.0, 122.0, 94.0, 98.0, 121.0, 118.0, 83.0, 96.0, 99.0, 123.0, 108.0, 107.0, 108.0, 93.0, 89.0, 101.0, 121.0, 121.0, 113.0, 108.0, 83.0, 123.0, 89.0, 105.0, 99.0, 100.0, 108.0, 105.0, 95.0, 112.0, 101.0, 110.0, 93.0, 108.0, 94.0, 120.0, 118.0, 107.0, 98.0, 121.0, 102.0, 97.0, 111.0, 126.0, 102.0, 108.0, 107.0, 108.0, 95.0, 97.0, 96.0, 118.0, 100.0, 111.0, 103.0, 92.0, 100.0, 101.0, 100.0, 103.0, 112.0, 87.0, 86.0, 119.0, 97.0, 101.0, 119.0, 120.0, 124.0, 114.0, 108.0, 105.0, 101.0, 104.0, 103.0, 98.0, 86.0, 101.0, 115.0, 98.0, 90.0, 108.0, 102.0, 102.0, 108.0, 125.0, 109.0, 90.0, 115.0, 94.0, 114.0, 113.0, 98.0, 113.0, 122.0, 101.0, 97.0, 109.0, 106.0, 105.0, 115.0, 95.0, 117.0, 118.0, 95.0, 111.0, 88.0, 121.0, 121.0, 117.0, 138.0, 134.0, 89.0, 99.0, 117.0, 93.0, 106.0, 123.0, 117.0, 107.0, 117.0, 108.0, 86.0, 121.0, 125.0, 105.0, 114.0, 107.0, 129.0, 114.0, 114.0, 107.0, 120.0, 118.0, 101.0, 109.0, 107.0, 124.0, 120.0, 116.0, 103.0, 127.0, 126.0, 90.0, 102.0, 114.0, 111.0, 108.0, 136.0, 107.0, 112.0, 104.0, 113.0, 117.0, 133.0, 104.0, 125.0, 119.0, 111.0, 122.0, 100.0, 118.0, 119.0, 104.0, 85.0, 133.0, 104.0, 119.0, 118.0, 95.0, 117.0, 123.0, 101.0, 132.0, 121.0, 110.0, 116.0, 116.0, 111.0, 91.0, 104.0, 104.0, 115.0, 124.0, 105.0, 104.0, 105.0, 101.0, 99.0, 112.0, 126.0, 139.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.0215, 180.0215, 180.0215, 180.02148, 180.02148, 180.02147, 180.02145, 180.02135, 180.02115, 180.02094, 180.01961, 180.01897, 180.01846, 180.01622, 180.01544, 180.01474, 180.01422, 180.01404, 180.01433, 180.01431, 180.01425, 180.01393, 180.01378, 180.01398, 180.01486, 180.01613, 180.01736, 180.01823, 180.01926, 180.02052, 180.02249, 180.0247, 180.0269, 180.02905, 180.03157, 180.03452, 180.03809, 180.04181, 180.04561, 180.04982, 180.05472, 180.06001, 180.06567, 180.07184, 180.0788, 180.08618, 180.09402, 180.10249, 180.11177, 180.12202, 180.13301, 180.14465, 180.15689, 180.16972, 180.18321, 180.19737, 180.21191, 180.22699, 180.24295, 180.26004, 180.27771, 180.29611, 180.31612, 180.33702, 180.35811, 180.38084, 180.40419, 180.4287, 180.45442, 180.48056, 180.50702, 180.53406, 180.56171, 180.58975, 180.61829, 180.64751, 180.67677, 180.70682, 180.73743, 180.76886, 180.80061, 180.83215, 180.86478, 180.89844, 180.93239, 180.96716, 181.00246, 181.03769, 181.07275, 181.10832, 181.14499, 181.18263, 181.21957, 181.25639, 181.29378, 181.33115, 181.36745, 181.40192, 181.43672, 181.47206, 181.50702, 181.54108, 181.57564, 181.61107, 181.64665, 181.68359, 181.72212, 181.76016, 181.79727, 181.83466, 181.87212, 181.91078, 181.94928, 181.98863, 182.02866, 182.0679, 182.10756, 182.14766, 182.18661, 182.22534, 182.26395, 182.30188, 182.33997, 182.3786, 182.41617, 182.45273, 182.48906, 182.52652, 182.56755, 182.60834, 182.64743, 182.68629, 182.72655, 182.76643, 182.80617, 182.84549, 182.8847, 182.92358, 182.96255, 183.00255, 183.04317, 183.08311, 183.12239, 183.16113, 183.20087, 183.24062, 183.27989, 183.31709, 183.35413, 183.39204, 183.42976, 183.46664, 183.50266, 183.5378, 183.57317, 183.60986, 183.64481, 183.67638, 183.7079, 183.74036, 183.77179, 183.80507, 183.8432, 183.8837, 183.92522, 183.96664, 184.00832, 184.04984, 184.09091, 184.13011, 184.16745, 184.20192, 184.2364, 184.27042, 184.30766, 184.34671, 184.38367, 184.41844, 184.45454, 184.49117, 184.52921, 184.56746, 184.60696, 184.64819, 184.69025, 184.73074, 184.77034, 184.80975, 184.84845, 184.88777, 184.92712, 184.96806, 185.00996, 185.0508, 185.09145, 185.13165, 185.17198, 185.21196, 185.25362, 185.29736, 185.33859, 185.37759, 185.41449, 185.45093, 185.48775, 185.52527, 185.56303, 185.60017, 185.63844, 185.67694, 185.717, 185.75711, 185.79745, 185.83626, 185.87444, 185.91074, 185.94763, 185.98566, 186.02451, 186.06494, 186.10443, 186.14497, 186.18584, 186.22533, 186.26512, 186.30524, 186.34587, 186.38719, 186.42752, 186.46732, 186.5069, 186.54416, 186.58186, 186.62146, 186.66272, 186.7025, 186.74118, 186.78197, 186.82381, 186.86591, 186.90703, 186.94699, 186.98782, 187.02896, 187.07161, 187.11592, 187.16006, 187.20297, 187.24727, 187.29167, 187.33688, 187.38315, 187.43051, 187.47704, 187.52306, 187.56926, 187.61435, 187.65848, 187.70207, 187.74612, 187.791, 187.83688, 187.88379, 187.93002, 187.97664, 188.02202, 188.06602, 188.10904, 188.15352, 188.19698, 188.23994, 188.28452, 188.3309, 188.37823, 188.4254, 188.47156, 188.51752, 188.5639, 188.60988, 188.65466, 188.69901, 188.74353, 188.78758, 188.82999, 188.87415, 188.91789, 188.9626, 189.00793, 189.05475, 189.10188, 189.14818, 189.1933, 189.23761, 189.28363, 189.33023, 189.37675, 189.42268, 189.46941, 189.51593, 189.56395, 189.61171, 189.65927, 189.70778, 189.75581, 189.80321, 189.8503, 189.89809, 189.9472, 189.9967, 190.04593, 190.09396, 190.14343, 190.1933, 190.24219, 190.29274, 190.34343, 190.39359, 190.44443, 190.49617, 190.54893, 190.60107, 190.65158, 190.70294, 190.75449, 190.80663, 190.86197, 190.91545, 190.96892, 191.02086, 191.07315, 191.12288, 191.17188, 191.22237, 191.27545, 191.32816, 191.38139, 191.43503, 191.48665, 191.53937, 191.58943, 191.64163, 191.69427, 191.74928, 191.8026, 191.85596, 191.90891, 191.96182, 192.01491, 192.06815, 192.12227, 192.17641, 192.23074, 192.28561, 192.34024, 192.39484, 192.44731, 192.50171, 192.55782, 192.61383, 192.67009, 192.72624, 192.78252, 192.83763, 192.89287, 192.94981, 193.00703, 193.06404, 193.12177, 193.17989, 193.23723, 193.29391, 193.34985, 193.40605, 193.45912, 193.51132, 193.56346, 193.61696, 193.67215, 193.72841, 193.78329, 193.83797, 193.89262, 193.94887, 194.00604, 194.064, 194.12062, 194.17807, 194.23741, 194.29666, 194.35547, 194.41553, 194.47499, 194.53378, 194.59259, 194.65202, 194.70923, 194.76607, 194.82375, 194.88065, 194.93935]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.0215, 180.0215, 180.0215, 180.02148, 180.02148, 180.02147, 180.02145, 180.02135, 180.02115, 180.02094, 180.01961, 180.01897, 180.01846, 180.01622, 180.01544, 180.01474, 180.01422, 180.01404, 180.01433, 180.01431, 180.01425, 180.01393, 180.01378, 180.01398, 180.01486, 180.01613, 180.01736, 180.01823, 180.01926, 180.02052, 180.02249, 180.0247, 180.0269, 180.02905, 180.03157, 180.03452, 180.03809, 180.04181, 180.04561, 180.04982, 180.05472, 180.06001, 180.06567, 180.07184, 180.0788, 180.08618, 180.09402, 180.10249, 180.11177, 180.12202, 180.13301, 180.14465, 180.15689, 180.16972, 180.18321, 180.19737, 180.21191, 180.22699, 180.24295, 180.26004, 180.27771, 180.29611, 180.31612, 180.33702, 180.35811, 180.38084, 180.40419, 180.4287, 180.45442, 180.48056, 180.50702, 180.53406, 180.56171, 180.58975, 180.61829, 180.64751, 180.67677, 180.70682, 180.73743, 180.76886, 180.80061, 180.83215, 180.86478, 180.89844, 180.93239, 180.96716, 181.00246, 181.03769, 181.07275, 181.10832, 181.14499, 181.18263, 181.21957, 181.25639, 181.29378, 181.33115, 181.36745, 181.40192, 181.43672, 181.47206, 181.50702, 181.54108, 181.57564, 181.61107, 181.64665, 181.68359, 181.72212, 181.76016, 181.79727, 181.83466, 181.87212, 181.91078, 181.94928, 181.98863, 182.02866, 182.0679, 182.10756, 182.14766, 182.18661, 182.22534, 182.26395, 182.30188, 182.33997, 182.3786, 182.41617, 182.45273, 182.48906, 182.52652, 182.56755, 182.60834, 182.64743, 182.68629, 182.72655, 182.76643, 182.80617, 182.84549, 182.8847, 182.92358, 182.96255, 183.00255, 183.04317, 183.08311, 183.12239, 183.16113, 183.20087, 183.24062, 183.27989, 183.31709, 183.35413, 183.39204, 183.42976, 183.46664, 183.50266, 183.5378, 183.57317, 183.60986, 183.64481, 183.67638, 183.7079, 183.74036, 183.77179, 183.80507, 183.8432, 183.8837, 183.92522, 183.96664, 184.00832, 184.04984, 184.09091, 184.13011, 184.16745, 184.20192, 184.2364, 184.27042, 184.30766, 184.34671, 184.38367, 184.41844, 184.45454, 184.49117, 184.52921, 184.56746, 184.60696, 184.64819, 184.69025, 184.73074, 184.77034, 184.80975, 184.84845, 184.88777, 184.92712, 184.96806, 185.00996, 185.0508, 185.09145, 185.13165, 185.17198, 185.21196, 185.25362, 185.29736, 185.33859, 185.37759, 185.41449, 185.45093, 185.48775, 185.52527, 185.56303, 185.60017, 185.63844, 185.67694, 185.717, 185.75711, 185.79745, 185.83626, 185.87444, 185.91074, 185.94763, 185.98566, 186.02451, 186.06494, 186.10443, 186.14497, 186.18584, 186.22533, 186.26512, 186.30524, 186.34587, 186.38719, 186.42752, 186.46732, 186.5069, 186.54416, 186.58186, 186.62146, 186.66272, 186.7025, 186.74118, 186.78197, 186.82381, 186.86591, 186.90703, 186.94699, 186.98782, 187.02896, 187.07161, 187.11592, 187.16006, 187.20297, 187.24727, 187.29167, 187.33688, 187.38315, 187.43051, 187.47704, 187.52306, 187.56926, 187.61435, 187.65848, 187.70207, 187.74612, 187.791, 187.83688, 187.88379, 187.93002, 187.97664, 188.02202, 188.06602, 188.10904, 188.15352, 188.19698, 188.23994, 188.28452, 188.3309, 188.37823, 188.4254, 188.47156, 188.51752, 188.5639, 188.60988, 188.65466, 188.69901, 188.74353, 188.78758, 188.82999, 188.87415, 188.91789, 188.9626, 189.00793, 189.05475, 189.10188, 189.14818, 189.1933, 189.23761, 189.28363, 189.33023, 189.37675, 189.42268, 189.46941, 189.51593, 189.56395, 189.61171, 189.65927, 189.70778, 189.75581, 189.80321, 189.8503, 189.89809, 189.9472, 189.9967, 190.04593, 190.09396, 190.14343, 190.1933, 190.24219, 190.29274, 190.34343, 190.39359, 190.44443, 190.49617, 190.54893, 190.60107, 190.65158, 190.70294, 190.75449, 190.80663, 190.86197, 190.91545, 190.96892, 191.02086, 191.07315, 191.12288, 191.17188, 191.22237, 191.27545, 191.32816, 191.38139, 191.43503, 191.48665, 191.53937, 191.58943, 191.64163, 191.69427, 191.74928, 191.8026, 191.85596, 191.90891, 191.96182, 192.01491, 192.06815, 192.12227, 192.17641, 192.23074, 192.28561, 192.34024, 192.39484, 192.44731, 192.50171, 192.55782, 192.61383, 192.67009, 192.72624, 192.78252, 192.83763, 192.89287, 192.94981, 193.00703, 193.06404, 193.12177, 193.17989, 193.23723, 193.29391, 193.34985, 193.40605, 193.45912, 193.51132, 193.56346, 193.61696, 193.67215, 193.72841, 193.78329, 193.83797, 193.89262, 193.94887, 194.00604, 194.064, 194.12062, 194.17807, 194.23741, 194.29666, 194.35547, 194.41553, 194.47499, 194.53378, 194.59259, 194.65202, 194.70923, 194.76607, 194.82375, 194.88065, 194.93935]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [25.13033, 1.48166, 1.46987, 1.47023, 1.48503, 1.46592, 1.47336, 1.47508, 1.47402, 1.4685, 1.46594, 1.46551, 1.47349, 1.47267, 1.46624, 1.4694, 1.46787, 1.46277, 1.47132, 1.47851, 1.46741, 1.46542, 1.4696, 1.47275, 1.46461, 1.47691, 1.4675, 1.4656, 1.47118, 1.46861, 1.46276, 1.46336, 1.46191, 1.46454, 1.46661, 1.45397, 1.45433, 1.45318, 1.47248, 1.45987, 1.4605, 1.47021, 1.46471, 1.46712, 1.47916, 1.46564, 1.46806, 1.48231, 1.47331, 1.47647, 1.4749, 1.47736, 1.47088, 1.48046, 1.47029, 1.4749, 1.47423, 1.4743, 1.47451, 1.47312, 1.46669, 1.48162, 1.47248, 1.47813, 1.47924, 1.47693, 1.4857, 1.47407, 1.47761, 1.47904, 1.47169, 1.46697, 1.48901, 1.47837, 1.47292, 1.48078, 1.49273, 1.48823, 1.48311, 1.48576, 1.48783, 1.48617, 1.47144, 1.46991, 1.46885, 1.47351, 1.47373, 1.46882, 1.46809, 1.46714, 1.4672, 1.47772, 1.46612, 1.46651, 1.47094, 1.47578, 1.46913, 1.48331, 1.4865, 1.48787, 1.47171, 1.46821, 1.4802, 1.46723, 1.47379, 1.46841, 1.46785, 1.47559, 1.47509, 1.46854, 1.47345, 1.47159, 1.46793, 1.47819, 1.48813, 1.4716, 1.47495, 1.46872, 1.47829, 1.47064, 1.47018, 1.47559, 1.47576, 1.47037, 1.47433, 1.47533, 1.47013, 1.47921, 1.47494, 1.4767, 1.47607, 1.47345, 1.47128, 1.47431, 1.46759, 1.46948, 1.46669, 1.47222, 1.46674, 1.47388, 1.47388, 1.46524, 1.47407, 1.47207, 1.46963, 1.47611, 1.47057, 1.47046, 1.47507, 1.4718, 1.47093, 1.46875, 1.47966, 1.47691, 1.47958, 1.46848, 1.47659, 1.47233, 1.46829, 1.47134, 1.47162, 1.47084, 1.46812, 1.46169, 1.47005, 1.47196, 1.47131, 1.4779, 1.47053, 1.46873, 1.47177, 1.47562, 1.47441, 1.47279, 1.4738, 1.47473, 1.47647, 1.4711, 1.47612, 1.47591, 1.48126, 1.47512, 1.47351, 1.47769, 1.46263, 1.47234, 1.47526, 1.47224, 1.47085, 1.46942, 1.46803, 1.4759, 1.47343, 1.46362, 1.4685, 1.47079, 1.47101, 1.47158, 1.47044, 1.46992, 1.46298, 1.47836, 1.46169, 1.46751, 1.47839, 1.47255, 1.47103, 1.47052, 1.46863, 1.4668, 1.4769, 1.47204, 1.4723, 1.47157, 1.4667, 1.47441, 1.48003, 1.47181, 1.48009, 1.48373, 1.47652, 1.4796, 1.47353, 1.47567, 1.47796, 1.47632, 1.48009, 1.4717, 1.47188, 1.48104, 1.47363, 1.47129, 1.47793, 1.47574, 1.47484, 1.47619, 1.47177, 1.47614, 1.47933, 1.47156, 1.46844, 1.4802, 1.47829, 1.47093, 1.4754, 1.47276, 1.57859, 1.4684, 1.47537, 1.54583, 1.47639, 1.57948, 1.47918, 1.48066, 1.48212, 1.4774, 1.47852, 1.47639, 1.47826, 1.48039, 1.4739, 1.4819, 1.48028, 1.47407, 1.47624, 1.48205, 1.47628, 1.48393, 1.48589, 1.47517, 1.47758, 1.47729, 1.48745, 1.47685, 1.48033, 1.47602, 1.47812, 1.48054, 1.47432, 1.47337, 1.47804, 1.47123, 1.47425, 1.47715, 1.47794, 1.47273, 1.47454, 1.47875, 1.4782, 1.47577, 1.47167, 1.47763, 1.4744, 1.47683, 1.48168, 1.47497, 1.47434, 1.4796, 1.4776, 1.47214, 1.47435, 1.47766, 1.4835, 1.48072, 1.4744, 1.48392, 1.47533, 1.47683, 1.47742, 1.48516, 1.47634, 1.478, 1.47244, 1.48265, 1.47422, 1.48296, 1.48311, 1.47628, 1.47751, 1.48129, 1.47507, 1.48075, 1.47775, 1.47657, 1.48203, 1.48345, 1.48818, 1.48194, 1.48374, 1.482, 1.48749, 1.48551, 1.48527, 1.4871, 1.49114, 1.48723, 1.47874, 1.47877, 1.48314, 1.47745, 1.47138, 1.4823, 1.4909, 1.48278, 1.48582, 1.48063, 1.47195, 1.47501, 1.47117, 1.47685, 1.47555, 1.47306, 1.54386, 1.47358, 1.57973, 1.47563, 1.47575, 1.56224, 1.47774, 1.4817, 1.48012, 1.48778, 1.47737, 1.47738, 1.48069, 1.47712, 1.47909, 1.47385, 1.47532, 1.47459, 1.47167, 1.47808, 1.48123, 1.47993, 1.46614, 1.46983, 1.47318, 1.47539, 1.47425, 1.47523, 1.47895, 1.47481, 1.4698, 1.46941, 1.47466, 1.47011, 1.46611, 1.47663, 1.47626, 1.4741, 1.47847, 1.46407, 1.47268, 1.47738, 1.46488, 1.48113, 1.47284, 1.46934, 1.47784, 1.4777]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.6001]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.6001]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [270.45398]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [270.45398]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/model_config.yaml
index 0efe0da30b..743064e121 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp/model_config.yaml
@@ -1,6 +1,7 @@
 ENV_VARS:
   CUDA_DEVICE_MAX_CONNECTIONS: 1
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
 MODEL_ARGS:
@@ -45,7 +46,7 @@ MODEL_ARGS:
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/golden_values.json
new file mode 100644
index 0000000000..0af59da700
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [21.65799, 1.57316, 1.56036, 1.56197, 1.56002, 1.57036, 1.57498, 1.57179, 1.57223, 1.56447, 1.57065, 1.57253, 1.56833, 1.57388, 1.58074, 1.57741, 1.58388, 1.58795, 1.5903, 1.58075, 1.57656, 1.58312, 1.57306, 1.57348, 1.58999, 1.57118, 1.56942, 1.57642, 1.58455, 1.57798, 1.57753, 1.5848, 1.57952, 1.57466, 1.5634, 1.5759, 1.57055, 1.56518, 1.64863, 1.56915, 1.57234, 1.57176, 1.59307, 1.58513, 1.59397, 1.59455, 1.58862, 1.58627, 1.57781, 1.5836, 1.59175, 1.58787, 1.58531, 1.56743, 1.56768, 1.57061, 1.57416, 1.56759, 1.5696, 1.57589, 1.57313, 1.571, 1.58684, 1.58081, 1.58172, 1.57572, 1.58332, 1.58369, 1.5742, 1.58521, 1.57857, 1.57985, 1.59598, 1.58564, 1.58954, 1.58921, 1.58516, 1.58693, 1.58278, 1.58855, 1.58036, 1.58425, 1.57404, 1.56846, 1.57061, 1.57471, 1.57444, 1.57552, 1.58566, 1.59602, 1.57809, 1.59795, 1.58523, 1.58552, 1.58948, 1.5857, 1.58918, 1.58406, 1.58274, 1.58292, 1.5878, 1.57929, 1.57852, 1.57229, 1.58645, 1.58337, 1.57647, 1.56993, 1.57461, 1.57583, 1.57981, 1.58228, 1.58026, 1.58041, 1.57147, 1.57774, 1.57198, 1.56711, 1.56216, 1.57948, 1.57013, 1.5652, 1.57538, 1.59385, 1.58672, 1.57603, 1.57508, 1.58044, 1.56643, 1.57319, 1.56412, 1.56703, 1.57342, 1.57169, 1.58538, 1.57905, 1.57735, 1.5713, 1.56908, 1.56945, 1.57129, 1.5672, 1.57775, 1.58937, 1.59019, 1.5751, 1.58049, 1.58855, 1.58446, 1.59003, 1.58787, 1.58871, 1.59524, 1.59317, 1.59223, 1.59165, 1.58901, 1.59193, 1.5866, 1.59184, 1.59323, 1.59575, 1.58596, 1.59591, 1.58463, 1.58779, 1.59392, 1.59398, 1.59893, 1.5974, 1.59446, 1.58691, 1.58241, 1.58352, 1.59639, 1.58013, 1.59181, 1.58597, 1.58425, 1.58787, 1.58445, 1.58197, 1.58869, 1.5852, 1.58751, 1.5889, 1.58458, 1.57701, 1.58666, 1.584, 1.57776, 1.58858, 1.58222, 1.58721, 1.60018, 1.59115, 1.59271, 1.58842, 1.59023, 1.58933, 1.57882, 1.59135, 1.5868, 1.57554, 1.58258, 1.58243, 1.58389, 1.58426, 1.5849, 1.58819, 1.58199, 1.58031, 1.58504, 1.58277, 1.5863, 1.57949, 1.58628, 1.58781, 1.58443, 1.57924, 1.58531, 1.59139, 1.58724, 1.58582, 1.59165, 1.58221, 1.58782, 1.59196, 1.58549, 1.58279, 1.59669, 1.58729, 1.58776, 1.58434, 1.58643, 1.57486, 1.58484, 1.57875, 1.58178, 1.58296, 1.57564, 1.57269, 1.73935, 1.63419, 1.58507, 1.59194, 1.5809, 1.60067, 1.59666, 1.59408, 1.59512, 1.68832, 1.59093, 1.57923, 1.58167, 1.5802, 1.58149, 1.59105, 1.58674, 1.59021, 1.59488, 1.60007, 1.59231, 1.59296, 1.59159, 1.588, 1.58471, 1.58515, 1.58686, 1.58415, 1.58593, 1.58185, 1.58805, 1.59063, 1.58623, 1.58868, 1.5863, 1.58712, 1.58387, 1.58919, 1.58738, 1.58618, 1.58901, 1.58673, 1.5896, 1.59327, 1.58995, 1.59034, 1.59043, 1.58508, 1.58835, 1.59575, 1.59028, 1.58788, 1.59495, 1.59031, 1.58998, 1.58896, 1.59037, 1.58923, 1.59259, 1.59082, 1.59843, 1.59394, 1.59716, 1.58592, 1.58443, 1.59841, 1.58588, 1.59009, 1.58471, 1.58793, 1.59585, 1.58806, 1.59097, 1.59974, 1.58594, 1.59971, 1.5913, 1.5727, 1.57474, 1.58074, 1.57644, 1.58641, 1.58808, 1.58075, 1.5907, 1.58838, 1.58642, 1.58856, 1.58469, 1.58982, 1.59264, 1.59172, 1.58848, 1.59119, 1.59145, 1.58124, 1.60003, 1.58841, 1.59199, 1.58955, 1.59024, 1.58713, 1.58159, 1.58812, 1.58697, 1.59477, 1.58735, 1.68808, 1.60409, 1.59368, 1.68921, 1.59656, 1.59503, 1.59737, 1.5981, 1.6072, 1.60584, 1.60205, 1.60339, 1.59005, 1.59398, 1.59059, 1.5983, 1.59588, 1.58451, 1.59372, 1.59209, 1.58828, 1.59305, 1.59272, 1.59217, 1.59417, 1.59371, 1.60293, 1.6081, 1.59666, 1.59861, 1.59979, 1.59362, 1.60255, 1.60302, 1.60884, 1.60587, 1.5947, 1.59209, 1.60211, 1.60023, 1.60283, 1.60565, 1.6008, 1.5957, 1.60008, 1.59899, 1.59865, 1.59781, 1.59196, 1.59478, 1.59227]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.22042, 0.7887, 0.79083, 0.78962, 0.78756, 0.78885, 0.8016, 0.80118, 0.79635, 0.79549, 0.79171, 0.803, 0.8016, 0.79277, 0.79347, 0.80205, 0.80724, 0.8102, 0.80595, 0.79227, 0.78683, 0.79736, 0.79666, 0.79876, 0.80245, 0.79592, 0.79874, 0.79753, 0.81164, 0.79672, 0.79701, 0.80746, 0.80543, 0.79696, 0.79511, 0.79932, 0.79557, 0.79429, 0.84751, 0.79126, 0.79445, 0.79427, 0.81209, 0.80591, 0.79877, 0.8166, 0.8125, 0.80956, 0.80732, 0.79604, 0.80371, 0.80021, 0.79673, 0.78625, 0.79742, 0.79855, 0.79833, 0.79792, 0.79392, 0.79627, 0.78993, 0.80003, 0.78776, 0.80568, 0.77968, 0.7912, 0.79925, 0.79922, 0.79071, 0.79884, 0.78877, 0.79858, 0.81252, 0.8067, 0.79219, 0.81833, 0.81779, 0.80094, 0.80137, 0.81945, 0.80719, 0.79232, 0.79516, 0.80871, 0.80104, 0.79685, 0.80162, 0.80637, 0.80248, 0.80857, 0.81037, 0.80869, 0.7965, 0.80743, 0.8098, 0.80128, 0.80589, 0.80206, 0.80032, 0.80015, 0.79522, 0.79329, 0.80165, 0.80384, 0.80062, 0.79949, 0.80381, 0.78559, 0.80393, 0.80321, 0.80107, 0.79216, 0.79542, 0.79246, 0.80303, 0.8106, 0.79065, 0.79761, 0.79846, 0.80131, 0.80281, 0.79732, 0.7963, 0.81465, 0.81139, 0.79778, 0.80117, 0.79101, 0.78623, 0.79644, 0.7976, 0.79653, 0.79953, 0.79765, 0.80015, 0.81095, 0.80579, 0.7998, 0.7917, 0.79794, 0.79775, 0.79275, 0.80199, 0.81948, 0.81204, 0.79625, 0.79973, 0.79652, 0.80445, 0.80534, 0.80518, 0.79884, 0.81423, 0.80952, 0.81247, 0.80766, 0.80443, 0.81182, 0.80591, 0.81339, 0.80677, 0.79581, 0.79801, 0.81209, 0.7963, 0.79413, 0.8031, 0.80814, 0.80927, 0.81215, 0.81255, 0.79604, 0.80852, 0.80814, 0.81295, 0.80402, 0.81318, 0.8097, 0.80155, 0.81294, 0.81295, 0.80384, 0.81085, 0.80809, 0.81049, 0.81462, 0.81121, 0.80114, 0.81317, 0.8073, 0.80801, 0.81335, 0.81351, 0.81644, 0.8235, 0.8092, 0.81494, 0.80197, 0.80738, 0.80524, 0.80729, 0.81006, 0.81098, 0.8058, 0.81736, 0.81018, 0.81686, 0.81077, 0.81584, 0.81737, 0.81149, 0.81076, 0.81213, 0.8138, 0.81013, 0.80497, 0.82135, 0.81652, 0.81154, 0.81448, 0.81949, 0.81162, 0.81162, 0.80853, 0.81191, 0.81703, 0.8125, 0.80932, 0.80851, 0.79798, 0.81183, 0.80938, 0.80838, 0.81083, 0.81336, 0.81205, 0.81618, 0.80587, 0.81362, 0.81042, 0.80604, 0.80513, 0.95515, 0.83951, 0.81274, 0.80912, 0.80158, 0.81243, 0.81495, 0.81427, 0.81731, 0.90437, 0.812, 0.81127, 0.80335, 0.80701, 0.81174, 0.81789, 0.8062, 0.81818, 0.81364, 0.82457, 0.81861, 0.81831, 0.81451, 0.81624, 0.819, 0.81664, 0.81149, 0.81897, 0.82098, 0.80639, 0.82356, 0.81998, 0.82291, 0.8172, 0.81813, 0.82015, 0.82009, 0.8243, 0.82188, 0.82103, 0.81895, 0.8227, 0.81898, 0.81687, 0.82231, 0.82276, 0.82281, 0.81752, 0.81589, 0.81308, 0.81283, 0.8171, 0.82039, 0.81907, 0.81497, 0.81934, 0.81714, 0.8101, 0.8135, 0.81914, 0.82468, 0.81829, 0.82195, 0.81334, 0.81505, 0.83, 0.82284, 0.82566, 0.82499, 0.82531, 0.81828, 0.81665, 0.82509, 0.82012, 0.82215, 0.82179, 0.81542, 0.80285, 0.81044, 0.80469, 0.8102, 0.8158, 0.81485, 0.82051, 0.80883, 0.82724, 0.81536, 0.8108, 0.81338, 0.81843, 0.81932, 0.81808, 0.81079, 0.81136, 0.82409, 0.81369, 0.81194, 0.81256, 0.81683, 0.81111, 0.8172, 0.80945, 0.80932, 0.8134, 0.81086, 0.81202, 0.81131, 0.86018, 0.81312, 0.81026, 0.91292, 0.81781, 0.81732, 0.82904, 0.82523, 0.83411, 0.83407, 0.83166, 0.82856, 0.81239, 0.81494, 0.82555, 0.83157, 0.82113, 0.80701, 0.81497, 0.8215, 0.80867, 0.81134, 0.82362, 0.81971, 0.808, 0.80408, 0.81663, 0.82201, 0.81271, 0.82346, 0.82415, 0.81743, 0.8063, 0.80216, 0.80964, 0.8105, 0.8118, 0.81122, 0.81369, 0.81864, 0.82566, 0.81149, 0.80986, 0.81981, 0.81964, 0.82004, 0.80608, 0.81446, 0.81929, 0.8075, 0.80881]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.62942, 0.75097, 0.74, 0.74537, 0.74999, 0.75094, 0.74822, 0.74322, 0.74143, 0.74188, 0.75087, 0.75511, 0.75059, 0.75125, 0.75555, 0.7505, 0.76577, 0.75929, 0.75813, 0.75798, 0.75777, 0.75449, 0.75219, 0.76004, 0.76606, 0.74726, 0.75154, 0.75719, 0.75304, 0.75913, 0.75194, 0.76105, 0.75155, 0.75361, 0.75194, 0.74863, 0.75344, 0.75699, 0.76125, 0.76168, 0.75845, 0.75545, 0.76173, 0.76702, 0.76538, 0.76769, 0.75666, 0.75657, 0.75518, 0.75767, 0.75791, 0.75998, 0.76253, 0.75636, 0.75269, 0.75165, 0.75005, 0.74953, 0.7487, 0.76173, 0.75616, 0.75523, 0.77089, 0.75678, 0.76, 0.7504, 0.7563, 0.75155, 0.75497, 0.74943, 0.75435, 0.75485, 0.76133, 0.75829, 0.75424, 0.74885, 0.75032, 0.76341, 0.76306, 0.75225, 0.74967, 0.75803, 0.74607, 0.74997, 0.75189, 0.75522, 0.75126, 0.75345, 0.75402, 0.76221, 0.75573, 0.75879, 0.7447, 0.75592, 0.75875, 0.76088, 0.76149, 0.75471, 0.75716, 0.7483, 0.75544, 0.7486, 0.75419, 0.75681, 0.75858, 0.76287, 0.75413, 0.75433, 0.75404, 0.75102, 0.75167, 0.75697, 0.75394, 0.75963, 0.75308, 0.75609, 0.74811, 0.74816, 0.74646, 0.74523, 0.74868, 0.74707, 0.74934, 0.7508, 0.76531, 0.76133, 0.75869, 0.75454, 0.74851, 0.74933, 0.74654, 0.74315, 0.74234, 0.74764, 0.75289, 0.7578, 0.75618, 0.75315, 0.75232, 0.75728, 0.75011, 0.75412, 0.75242, 0.74889, 0.75119, 0.75527, 0.75085, 0.7583, 0.76477, 0.75215, 0.75071, 0.76072, 0.75986, 0.76825, 0.75337, 0.75661, 0.75384, 0.76056, 0.76054, 0.76494, 0.7674, 0.76549, 0.75611, 0.76183, 0.75053, 0.75482, 0.75715, 0.76983, 0.77042, 0.76028, 0.77021, 0.75151, 0.75914, 0.75118, 0.76133, 0.75325, 0.76558, 0.75951, 0.76119, 0.75926, 0.75073, 0.75384, 0.75883, 0.7634, 0.76168, 0.76652, 0.75731, 0.75344, 0.76068, 0.75369, 0.75137, 0.75963, 0.7697, 0.751, 0.77098, 0.75284, 0.75939, 0.75995, 0.75928, 0.75802, 0.75677, 0.76065, 0.75638, 0.75119, 0.76038, 0.75423, 0.75553, 0.75918, 0.75995, 0.75408, 0.76136, 0.74612, 0.75854, 0.75865, 0.7593, 0.75419, 0.75151, 0.75761, 0.76577, 0.75463, 0.74788, 0.75358, 0.76279, 0.76172, 0.76321, 0.75292, 0.75124, 0.75794, 0.76269, 0.76049, 0.75669, 0.7573, 0.75738, 0.75375, 0.76126, 0.75621, 0.75055, 0.75297, 0.75603, 0.75099, 0.75101, 0.74554, 0.83246, 0.7545, 0.75293, 0.75203, 0.75391, 0.7554, 0.75839, 0.75728, 0.76242, 0.75203, 0.75857, 0.7516, 0.75317, 0.75327, 0.75445, 0.7579, 0.753, 0.753, 0.75219, 0.75665, 0.75118, 0.75048, 0.74602, 0.74682, 0.75041, 0.74864, 0.75542, 0.74976, 0.74748, 0.75186, 0.75401, 0.75027, 0.74959, 0.75363, 0.74766, 0.75374, 0.751, 0.75381, 0.75069, 0.74504, 0.75077, 0.75083, 0.75402, 0.74825, 0.75092, 0.75145, 0.75314, 0.75502, 0.74951, 0.7579, 0.75347, 0.7511, 0.75538, 0.75696, 0.7579, 0.75511, 0.75693, 0.75306, 0.74836, 0.7533, 0.75717, 0.76271, 0.75482, 0.75341, 0.74896, 0.75096, 0.74632, 0.75083, 0.74516, 0.74075, 0.75065, 0.75718, 0.75375, 0.7557, 0.7462, 0.75504, 0.75655, 0.74982, 0.75081, 0.74949, 0.74808, 0.75239, 0.75544, 0.74273, 0.75537, 0.75449, 0.75109, 0.7469, 0.7528, 0.75193, 0.75171, 0.75366, 0.75959, 0.74847, 0.75215, 0.75052, 0.76098, 0.75632, 0.75747, 0.74845, 0.74437, 0.75406, 0.75357, 0.75105, 0.75484, 0.75765, 0.75917, 0.7582, 0.75622, 0.75762, 0.74952, 0.75592, 0.75778, 0.74829, 0.75888, 0.75085, 0.75064, 0.74667, 0.751, 0.75208, 0.75768, 0.74883, 0.75857, 0.7487, 0.75962, 0.76274, 0.75413, 0.75644, 0.75008, 0.75022, 0.75465, 0.76027, 0.75685, 0.7526, 0.7567, 0.75515, 0.75552, 0.75496, 0.75875, 0.76104, 0.77511, 0.77406, 0.768, 0.7781, 0.77247, 0.78055, 0.77825, 0.76677, 0.78188, 0.77415, 0.77114, 0.77225, 0.77049, 0.77717, 0.77115, 0.76807, 0.77259, 0.77472]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.20334, 0.0143, 0.01667, 0.01326, 0.01295, 0.01293, 0.01334, 0.01436, 0.01318, 0.01437, 0.01301, 0.01378, 0.01472, 0.01468, 0.01314, 0.01281, 0.01302, 0.01378, 0.01285, 0.01444, 0.01432, 0.01486, 0.01305, 0.01348, 0.01674, 0.01301, 0.01444, 0.01426, 0.01437, 0.01321, 0.01305, 0.01316, 0.01395, 0.01333, 0.01301, 0.01363, 0.01284, 0.01423, 0.01642, 0.01753, 0.01691, 0.01476, 0.01495, 0.01652, 0.01707, 0.02019, 0.01642, 0.01534, 0.01555, 0.01455, 0.01613, 0.01682, 0.01611, 0.01302, 0.01316, 0.01386, 0.0152, 0.01835, 0.01342, 0.01579, 0.01295, 0.01372, 0.01717, 0.0153, 0.01567, 0.01348, 0.01623, 0.0153, 0.01466, 0.01622, 0.01222, 0.01602, 0.02111, 0.01556, 0.01731, 0.01708, 0.01773, 0.0175, 0.01682, 0.0175, 0.01625, 0.0172, 0.01748, 0.02121, 0.01676, 0.01653, 0.01683, 0.01767, 0.01788, 0.01764, 0.01715, 0.02209, 0.01681, 0.01797, 0.01754, 0.01797, 0.01781, 0.01828, 0.0179, 0.01691, 0.01823, 0.0176, 0.01724, 0.0166, 0.01718, 0.01732, 0.0149, 0.01363, 0.01477, 0.01454, 0.01309, 0.01297, 0.01408, 0.0145, 0.01297, 0.01965, 0.01506, 0.01303, 0.01404, 0.01373, 0.01435, 0.01442, 0.01449, 0.01568, 0.01599, 0.01299, 0.01288, 0.01478, 0.01302, 0.01354, 0.01604, 0.01518, 0.01493, 0.01391, 0.01308, 0.01275, 0.01267, 0.01483, 0.0133, 0.01279, 0.01339, 0.01261, 0.01553, 0.01269, 0.0125, 0.01256, 0.01329, 0.0129, 0.01284, 0.01681, 0.01599, 0.01537, 0.0153, 0.01362, 0.01518, 0.01566, 0.01486, 0.01485, 0.01522, 0.01745, 0.01558, 0.01496, 0.01484, 0.01693, 0.01487, 0.01546, 0.02093, 0.01683, 0.01724, 0.01738, 0.01648, 0.01861, 0.01776, 0.01745, 0.01724, 0.01583, 0.02118, 0.01682, 0.01836, 0.02112, 0.01766, 0.0169, 0.01696, 0.01695, 0.01754, 0.01652, 0.0184, 0.0173, 0.01627, 0.01667, 0.01742, 0.01775, 0.01745, 0.01643, 0.01709, 0.01696, 0.01761, 0.01648, 0.01725, 0.01672, 0.21908, 0.01675, 0.01611, 0.01752, 0.01616, 0.01728, 0.01777, 0.0171, 0.01749, 0.01847, 0.01858, 0.01789, 0.01723, 0.01628, 0.01773, 0.01691, 0.01878, 0.01787, 0.0209, 0.01796, 0.01741, 0.01777, 0.01829, 0.01892, 0.01729, 0.01774, 0.01727, 0.02061, 0.01571, 0.01771, 0.01838, 0.01772, 0.0174, 0.01766, 0.01725, 0.01763, 0.01752, 0.01709, 0.01817, 0.02143, 0.0161, 0.01751, 0.09405, 0.06723, 0.01758, 0.01661, 0.02181, 0.02167, 0.01822, 0.01785, 0.01747, 0.01708, 0.01826, 0.01765, 0.01811, 0.01727, 0.01812, 0.01807, 0.01812, 0.01919, 0.01774, 0.01749, 0.01737, 0.01751, 0.01714, 0.02283, 0.01759, 0.01975, 0.02057, 0.01799, 0.01752, 0.01739, 0.01757, 0.01773, 0.01789, 0.01729, 0.01642, 0.01712, 0.0176, 0.01717, 0.01691, 0.01727, 0.01589, 0.01789, 0.0174, 0.0174, 0.01722, 0.01761, 0.01802, 0.0174, 0.02069, 0.0171, 0.01719, 0.01766, 0.01768, 0.01677, 0.01705, 0.01777, 0.01669, 0.02073, 0.01723, 0.01707, 0.01707, 0.01723, 0.01751, 0.01953, 0.0174, 0.0167, 0.01749, 0.01753, 0.01974, 0.01695, 0.01888, 0.01805, 0.01809, 0.01779, 0.0192, 0.01732, 0.01965, 0.01793, 0.01875, 0.01855, 0.01915, 0.01839, 0.01868, 0.01864, 0.01893, 0.01823, 0.01908, 0.01892, 0.01884, 0.01914, 0.02012, 0.01861, 0.02283, 0.01928, 0.01945, 0.01841, 0.01795, 0.01816, 0.0187, 0.01867, 0.01891, 0.02308, 0.0188, 0.01869, 0.01974, 0.02014, 0.02234, 0.0193, 0.01762, 0.01819, 0.0184, 0.01952, 0.01974, 0.01869, 0.0205, 0.018, 0.0183, 0.01719, 0.01915, 0.01879, 0.0194, 0.01781, 0.01856, 0.01773, 0.01734, 0.01914, 0.0169, 0.019, 0.01792, 0.01743, 0.02488, 0.01724, 0.01703, 0.01755, 0.01784, 0.01774, 0.01824, 0.01859, 0.02236, 0.01639, 0.0181, 0.01772, 0.01786, 0.01787, 0.01629, 0.01663, 0.01687, 0.01734, 0.01643, 0.0175, 0.0166, 0.01686, 0.0162, 0.01662, 0.02025, 0.01762, 0.01683, 0.01837]}, "forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [5.65416, 0.02537, 0.02635, 0.02461, 0.02504, 0.02484, 0.02542, 0.02517, 0.02613, 0.02496, 0.02499, 0.02526, 0.02517, 0.02669, 0.02527, 0.02523, 0.02555, 0.02514, 0.02531, 0.02544, 0.02502, 0.02866, 0.02534, 0.02519, 0.02546, 0.02642, 0.02449, 0.02505, 0.02448, 0.02468, 0.02481, 0.02534, 0.02569, 0.02662, 0.02525, 0.02575, 0.02553, 0.02468, 0.02518, 0.02486, 0.02617, 0.0262, 0.02498, 0.02481, 0.02556, 0.02544, 0.02525, 0.02507, 0.02521, 0.02526, 0.02607, 0.02518, 0.02513, 0.02559, 0.02488, 0.02586, 0.02585, 0.02611, 0.02926, 0.02566, 0.02649, 0.02556, 0.02541, 0.02684, 0.0255, 0.02555, 0.0255, 0.0255, 0.02545, 0.02694, 0.02533, 0.02962, 0.02527, 0.02528, 0.02579, 0.02515, 0.02509, 0.02553, 0.02514, 0.02532, 0.02535, 0.02565, 0.02505, 0.02564, 0.02529, 0.02581, 0.02662, 0.02629, 0.02709, 0.02508, 0.0255, 0.02567, 0.02579, 0.0251, 0.02471, 0.02553, 0.02567, 0.02524, 0.02526, 0.02542, 0.02549, 0.02485, 0.0254, 0.02557, 0.02563, 0.02532, 0.02527, 0.02538, 0.02679, 0.02564, 0.02917, 0.02565, 0.02736, 0.02515, 0.02504, 0.02493, 0.02534, 0.0255, 0.02468, 0.02576, 0.02535, 0.02502, 0.02542, 0.02937, 0.02618, 0.02564, 0.02552, 0.02493, 0.02464, 0.02534, 0.02541, 0.02506, 0.02906, 0.02585, 0.02551, 0.02458, 0.02524, 0.0254, 0.02487, 0.02705, 0.02476, 0.02422, 0.02846, 0.02862, 0.02919, 0.02491, 0.02528, 0.0255, 0.02536, 0.02481, 0.02663, 0.02537, 0.02529, 0.02555, 0.02495, 0.02532, 0.02892, 0.02477, 0.02508, 0.0255, 0.02505, 0.0255, 0.02603, 0.02601, 0.02543, 0.0257, 0.02514, 0.02658, 0.02696, 0.02519, 0.02558, 0.02777, 0.027, 0.02528, 0.02566, 0.02491, 0.02592, 0.02533, 0.02595, 0.0256, 0.02521, 0.02524, 0.02528, 0.02552, 0.02639, 0.02554, 0.02548, 0.02553, 0.02553, 0.02546, 0.02481, 0.02518, 0.02516, 0.02541, 0.02568, 0.02495, 0.02523, 0.02848, 0.02556, 0.02499, 0.022, 0.02884, 0.02809, 0.02537, 0.02485, 0.02541, 0.0241, 0.02529, 0.02531, 0.02522, 0.02532, 0.02491, 0.02523, 0.02501, 0.02691, 0.02738, 0.02935, 0.02585, 0.02542, 0.02516, 0.02571, 0.03013, 0.02563, 0.02483, 0.0253, 0.02509, 0.02525, 0.0255, 0.02513, 0.02517, 0.02489, 0.02524, 0.02485, 0.02507, 0.02536, 0.02583, 0.02534, 0.02509, 0.0251, 0.02531, 0.02518, 0.02475, 0.02917, 0.02567, 0.02587, 0.02568, 0.02609, 0.02628, 0.02622, 0.02564, 0.02497, 0.02578, 0.02549, 0.02526, 0.02494, 0.02571, 0.02582, 0.02631, 0.02647, 0.02581, 0.02643, 0.02664, 0.0263, 0.02556, 0.025, 0.02535, 0.02517, 0.02527, 0.0252, 0.02486, 0.02861, 0.02534, 0.02604, 0.02568, 0.02564, 0.02728, 0.02552, 0.02578, 0.02551, 0.02575, 0.02545, 0.02536, 0.02514, 0.02619, 0.02548, 0.02549, 0.02561, 0.02555, 0.02574, 0.02616, 0.02572, 0.02599, 0.02561, 0.02503, 0.02535, 0.02684, 0.02548, 0.02545, 0.02557, 0.02504, 0.02542, 0.0261, 0.02567, 0.02546, 0.0255, 0.02529, 0.02633, 0.03021, 0.0287, 0.0293, 0.0291, 0.03051, 0.03077, 0.02941, 0.03025, 0.02889, 0.02504, 0.02563, 0.02509, 0.02514, 0.02874, 0.02525, 0.02524, 0.02529, 0.02567, 0.02595, 0.02539, 0.02551, 0.02571, 0.02607, 0.02531, 0.02862, 0.02572, 0.02526, 0.02664, 0.02609, 0.02882, 0.02605, 0.02621, 0.02593, 0.02588, 0.02619, 0.02534, 0.02604, 0.02557, 0.02616, 0.02561, 0.02542, 0.02469, 0.02539, 0.02533, 0.02624, 0.02525, 0.02545, 0.02533, 0.02553, 0.02573, 0.02577, 0.0253, 0.02529, 0.02629, 0.02636, 0.02548, 0.02577, 0.0255, 0.02611, 0.02473, 0.02582, 0.02551, 0.02567, 0.0253, 0.02519, 0.0256, 0.02642, 0.02489, 0.02549, 0.02566, 0.0257, 0.02523, 0.02566, 0.02708, 0.02568, 0.025, 0.02826, 0.02772, 0.02446, 0.02415, 0.0242, 0.02452, 0.02402, 0.02491, 0.02511, 0.02443, 0.0247, 0.02457, 0.02433, 0.02427, 0.02485, 0.02473, 0.02411]}, "forward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.82565, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00019, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00015, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00018, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00014, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00014, 0.00014, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012]}, "backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02047, 0.0283, 0.02457, 0.02402, 0.02376, 0.02455, 0.02368, 0.02489, 0.03547, 0.02397, 0.02483, 0.02383, 0.02354, 0.02677, 0.02403, 0.02404, 0.02385, 0.02413, 0.02382, 0.02401, 0.02447, 0.02418, 0.02565, 0.02458, 0.02399, 0.02426, 0.02371, 0.02373, 0.02497, 0.02531, 0.02428, 0.02424, 0.02812, 0.02847, 0.02391, 0.0276, 0.02414, 0.02342, 0.02403, 0.0241, 0.02246, 0.0239, 0.02373, 0.02354, 0.024, 0.02551, 0.02523, 0.02434, 0.02333, 0.02695, 0.02802, 0.03335, 0.024, 0.02415, 0.02428, 0.0235, 0.02721, 0.02385, 0.02396, 0.02372, 0.02372, 0.02589, 0.02448, 0.02657, 0.02807, 0.02364, 0.02407, 0.02393, 0.02278, 0.02609, 0.02324, 0.02406, 0.02392, 0.02575, 0.02435, 0.02335, 0.02423, 0.02688, 0.02482, 0.02464, 0.0283, 0.02798, 0.02454, 0.02403, 0.02385, 0.02375, 0.024, 0.02436, 0.02658, 0.02418, 0.02444, 0.02438, 0.02772, 0.02445, 0.02469, 0.02482, 0.025, 0.0236, 0.02423, 0.02583, 0.02383, 0.02532, 0.02443, 0.02397, 0.02832, 0.02453, 0.02425, 0.02386, 0.02401, 0.02329, 0.02374, 0.02459, 0.02345, 0.02812, 0.02257, 0.02428, 0.03159, 0.02496, 0.02394, 0.02407, 0.02348, 0.02404, 0.0242, 0.02606, 0.02405, 0.02413, 0.02672, 0.02751, 0.02579, 0.02343, 0.02459, 0.02392, 0.02467, 0.02321, 0.02966, 0.02406, 0.02342, 0.02901, 0.02438, 0.02338, 0.02418, 0.02428, 0.02389, 0.02408, 0.02451, 0.02382, 0.02778, 0.02307, 0.02734, 0.02437, 0.02405, 0.02422, 0.02458, 0.02387, 0.02398, 0.02622, 0.0253, 0.02883, 0.02608, 0.02311, 0.02341, 0.0239, 0.02486, 0.02775, 0.02913, 0.02946, 0.03162, 0.03164, 0.03243, 0.02904, 0.03427, 0.02606, 0.02427, 0.02426, 0.02481, 0.02533, 0.02412, 0.02331, 0.02327, 0.02433, 0.02456, 0.02446, 0.02307, 0.02419, 0.02354, 0.02436, 0.02445, 0.02378, 0.02468, 0.02434, 0.02455, 0.02741, 0.02293, 0.02633, 0.02903, 0.02671, 0.02326, 0.0238, 0.02369, 0.02323, 0.02472, 0.02363, 0.02637, 0.02415, 0.0239, 0.02407, 0.02419, 0.0237, 0.02387, 0.02419, 0.02417, 0.02427, 0.02439, 0.02456, 0.02399, 0.02419, 0.0259, 0.02715, 0.02432, 0.02384, 0.02406, 0.02463, 0.02389, 0.02404, 0.02528, 0.02496, 0.0241, 0.02492, 0.02586, 0.02752, 0.02936, 0.02831, 0.02641, 0.02748, 0.02535, 0.0236, 0.02441, 0.02391, 0.02402, 0.02375, 0.02392, 0.02658, 0.02281, 0.02404, 0.02443, 0.02393, 0.02425, 0.02565, 0.02492, 0.02922, 0.02822, 0.02695, 0.02827, 0.02425, 0.02791, 0.02429, 0.02507, 0.02421, 0.02448, 0.02504, 0.02444, 0.02428, 0.02484, 0.02431, 0.0247, 0.02476, 0.02429, 0.02826, 0.02806, 0.02466, 0.02444, 0.02446, 0.02398, 0.0246, 0.02694, 0.02743, 0.02754, 0.02821, 0.02752, 0.02768, 0.02846, 0.02827, 0.02821, 0.02757, 0.02781, 0.03032, 0.0282, 0.02767, 0.02766, 0.02791, 0.02891, 0.02728, 0.02724, 0.02826, 0.02818, 0.0275, 0.02704, 0.02768, 0.02881, 0.02841, 0.02812, 0.02758, 0.02852, 0.02732, 0.02863, 0.0247, 0.02488, 0.02405, 0.02493, 0.02485, 0.025, 0.02485, 0.0248, 0.02492, 0.02512, 0.02464, 0.02467, 0.02816, 0.02752, 0.02469, 0.02368, 0.02464, 0.02438, 0.02448, 0.02474, 0.0246, 0.0247, 0.02471, 0.02492, 0.02452, 0.02459, 0.02436, 0.02461, 0.02714, 0.02468, 0.02624, 0.02941, 0.02449, 0.02703, 0.02762, 0.0284, 0.02681, 0.02872, 0.02442, 0.02456, 0.02406, 0.02457, 0.02358, 0.02347, 0.02871, 0.03113, 0.02849, 0.02643, 0.02442, 0.02499, 0.02477, 0.02568, 0.02464, 0.02487, 0.02408, 0.0248, 0.0262, 0.02523, 0.02571, 0.02565, 0.02504, 0.02409, 0.02564, 0.02393, 0.02423, 0.02644, 0.0241, 0.02354, 0.02445, 0.02479, 0.02481, 0.02499, 0.02444, 0.02433, 0.02438, 0.02439, 0.02468, 0.02426, 0.02465, 0.02263, 0.02673, 0.0262, 0.02622, 0.02641, 0.0272, 0.02655, 0.02722, 0.02659, 0.02705, 0.02744, 0.02687, 0.02797, 0.02579, 0.0241, 0.02442]}, "backward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00019, 0.00019, 0.00016, 0.0002, 0.00018, 0.00018, 0.00016, 0.00018, 0.00022, 0.00017, 0.00018, 0.00017, 0.00018, 0.00016, 0.00017, 0.00017, 0.00018, 0.00017, 0.00016, 0.00016, 0.00019, 0.00019, 0.00016, 0.00018, 0.00016, 0.00016, 0.00018, 0.00016, 0.00017, 0.00018, 0.00016, 0.00019, 0.00018, 0.00016, 0.00019, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00017, 0.00016, 0.00018, 0.00017, 0.00017, 0.00018, 0.00021, 0.00019, 0.00018, 0.00019, 0.00018, 0.00016, 0.00017, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.0002, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00016, 0.00019, 0.00021, 0.00017, 0.00016, 0.00016, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00019, 0.00019, 0.00016, 0.00018, 0.00021, 0.00017, 0.00018, 0.00017, 0.00018, 0.00016, 0.00016, 0.00018, 0.00017, 0.00016, 0.00018, 0.00036, 0.00016, 0.00022, 0.00016, 0.00016, 0.00019, 0.00019, 0.00018, 0.00017, 0.00016, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00017, 0.00018, 0.00019, 0.00018, 0.00018, 0.00032, 0.00018, 0.00018, 0.00016, 0.00021, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00021, 0.00016, 0.00019, 0.00019, 0.00018, 0.00017, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00016, 0.00018, 0.00019, 0.00018, 0.00021, 0.00016, 0.00017, 0.00016, 0.00016, 0.00017, 0.0002, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00019, 0.00027, 0.00031, 0.00017, 0.00017, 0.00016, 0.00016, 0.00019, 0.00018, 0.00016, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.0002, 0.0002, 0.00018, 0.00017, 0.00019, 0.00018, 0.00018, 0.00018, 0.00016, 0.00019, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00017, 0.00017, 0.00019, 0.00019, 0.00018, 0.00019, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.0002, 0.00016, 0.00018, 0.00016, 0.00017, 0.00018, 0.00018, 0.00019, 0.00018, 0.00016, 0.00018, 0.00019, 0.00018, 0.00016, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00019, 0.00016, 0.00018, 0.00017, 0.00019, 0.00037, 0.00017, 0.00017, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00019, 0.0002, 0.00016, 0.00018, 0.00029, 0.00019, 0.0002, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00017, 0.00037, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.0002, 0.00016, 0.00018, 0.00029, 0.00017, 0.00024, 0.00016, 0.00019, 0.00016, 0.00017, 0.00035, 0.00036, 0.00017, 0.00016, 0.0002, 0.00034, 0.0002, 0.00016, 0.00017, 0.0002, 0.00016, 0.00018, 0.00018, 0.00016, 0.00017, 0.00017, 0.00016, 0.00017, 0.00018, 0.00018, 0.00018, 0.00025, 0.00018, 0.00016, 0.00016, 0.00016, 0.00017, 0.00017, 0.00018, 0.00016, 0.00017, 0.00018, 0.00017, 0.00016, 0.00018, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00017, 0.00016, 0.00016, 0.00019, 0.00017, 0.00018, 0.00019, 0.00016, 0.00018, 0.00018, 0.00019, 0.00016, 0.00016, 0.00019, 0.00017, 0.00019, 0.00017, 0.00017, 0.00018, 0.00017, 0.00016, 0.00018, 0.00016, 0.00018, 0.00019, 0.00016, 0.00017, 0.00019, 0.00016, 0.00017, 0.00016, 0.00016, 0.00019, 0.00017, 0.00018, 0.00018, 0.00018, 0.00016, 0.00019, 0.00018, 0.00017, 0.00018, 0.00016, 0.00018, 0.0002, 0.00017, 0.00016, 0.00017, 0.00017, 0.00018, 0.00018, 0.00018, 0.00016, 0.00019, 0.00018, 0.00018, 0.00018, 0.00019, 0.00016, 0.00016, 0.00017, 0.00018, 0.00018, 0.00016]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [7.72045, 0.09004, 0.10467, 0.09849, 0.09238, 0.09943, 0.10332, 0.10911, 0.10563, 0.10498, 0.10272, 0.10382, 0.10192, 0.10289, 0.10891, 0.10722, 0.1057, 0.11565, 0.11445, 0.10746, 0.11354, 0.10514, 0.10376, 0.08937, 0.09262, 0.08764, 0.08288, 0.09035, 0.09702, 0.09008, 0.09616, 0.09645, 0.09564, 0.08936, 0.08325, 0.08878, 0.08887, 0.08097, 0.16157, 0.08262, 0.08896, 0.09145, 0.09803, 0.08184, 0.09702, 0.0971, 0.09683, 0.09764, 0.08935, 0.0971, 0.10578, 0.09846, 0.10251, 0.08742, 0.08778, 0.08971, 0.09353, 0.08897, 0.09, 0.08803, 0.08686, 0.08756, 0.09058, 0.08647, 0.08759, 0.09747, 0.10439, 0.10521, 0.09647, 0.10904, 0.09397, 0.09736, 0.10653, 0.0936, 0.10631, 0.1059, 0.10256, 0.09952, 0.09927, 0.10519, 0.10149, 0.09551, 0.10221, 0.10051, 0.09736, 0.09577, 0.0979, 0.09361, 0.09726, 0.10742, 0.0922, 0.10792, 0.10335, 0.10219, 0.1015, 0.09685, 0.09726, 0.10184, 0.09792, 0.10191, 0.1005, 0.10051, 0.09742, 0.09427, 0.09441, 0.08885, 0.09704, 0.09172, 0.09714, 0.09629, 0.10183, 0.09676, 0.09562, 0.09133, 0.09003, 0.10068, 0.09125, 0.0941, 0.09629, 0.10409, 0.09294, 0.09359, 0.10104, 0.10583, 0.09162, 0.08569, 0.08813, 0.093, 0.08756, 0.10008, 0.09688, 0.1054, 0.10747, 0.10112, 0.10023, 0.10296, 0.09747, 0.0945, 0.09503, 0.09075, 0.10094, 0.09821, 0.10359, 0.11126, 0.11094, 0.10686, 0.10472, 0.10387, 0.09679, 0.10627, 0.11005, 0.10858, 0.10916, 0.10819, 0.11254, 0.11227, 0.1067, 0.10979, 0.10635, 0.10862, 0.11093, 0.10588, 0.1078, 0.11054, 0.10333, 0.10314, 0.11111, 0.10133, 0.10064, 0.10338, 0.09919, 0.10252, 0.10368, 0.10692, 0.11169, 0.10373, 0.1082, 0.11025, 0.09905, 0.10905, 0.11343, 0.10499, 0.10807, 0.10315, 0.09841, 0.10583, 0.10804, 0.09746, 0.10771, 0.10609, 0.10625, 0.1058, 0.10401, 0.10832, 0.10595, 0.10705, 0.11742, 0.10139, 0.10969, 0.09952, 0.10696, 0.11066, 0.10165, 0.10114, 0.10538, 0.10594, 0.11402, 0.10492, 0.10645, 0.11173, 0.10848, 0.11309, 0.10714, 0.10786, 0.10722, 0.10193, 0.11309, 0.0997, 0.10535, 0.10927, 0.11186, 0.11523, 0.10176, 0.11174, 0.10738, 0.10339, 0.10818, 0.10428, 0.10357, 0.102, 0.11031, 0.10504, 0.10603, 0.10464, 0.10777, 0.10003, 0.11154, 0.10215, 0.10884, 0.1135, 0.10294, 0.10521, 0.18146, 0.15513, 0.10795, 0.10192, 0.09492, 0.1123, 0.11068, 0.10753, 0.10062, 0.20176, 0.10053, 0.10546, 0.10178, 0.10047, 0.10162, 0.10317, 0.10396, 0.10664, 0.11601, 0.12091, 0.11596, 0.11321, 0.11757, 0.11585, 0.1102, 0.10582, 0.10902, 0.11204, 0.11498, 0.11048, 0.11561, 0.12266, 0.11204, 0.10563, 0.11232, 0.10806, 0.10523, 0.11245, 0.10857, 0.10998, 0.10637, 0.11004, 0.10832, 0.1137, 0.11249, 0.1137, 0.11325, 0.10714, 0.10913, 0.11342, 0.10767, 0.11168, 0.1127, 0.10979, 0.10867, 0.10899, 0.11074, 0.10988, 0.11196, 0.11045, 0.10625, 0.10876, 0.11621, 0.10786, 0.11166, 0.1137, 0.1159, 0.12034, 0.12688, 0.13086, 0.12051, 0.11583, 0.12425, 0.12785, 0.11994, 0.1156, 0.11305, 0.1064, 0.11037, 0.11458, 0.10783, 0.11267, 0.11832, 0.11674, 0.12221, 0.11896, 0.11355, 0.12228, 0.11929, 0.11934, 0.11071, 0.11311, 0.12323, 0.11815, 0.1124, 0.10574, 0.10714, 0.11404, 0.1155, 0.11749, 0.11507, 0.11217, 0.11336, 0.11724, 0.11529, 0.11873, 0.11413, 0.11342, 0.11662, 0.11253, 0.21031, 0.1153, 0.11949, 0.12203, 0.12384, 0.12782, 0.12363, 0.12548, 0.12785, 0.11974, 0.12339, 0.11698, 0.1138, 0.11801, 0.11508, 0.12193, 0.1161, 0.11722, 0.11675, 0.12016, 0.12149, 0.12239, 0.12005, 0.12773, 0.12921, 0.11853, 0.11824, 0.12298, 0.11989, 0.12376, 0.12606, 0.12268, 0.12167, 0.11886, 0.10748, 0.11973, 0.11767, 0.12515, 0.11708, 0.11935, 0.12016, 0.12159, 0.11803, 0.11151, 0.11606, 0.11651, 0.12057, 0.10879]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3.17241, 0.01112, 0.01172, 0.00869, 0.00901, 0.01001, 0.01115, 0.00794, 0.00798, 0.0109, 0.01029, 0.01093, 0.01077, 0.01317, 0.01259, 0.00838, 0.01022, 0.00884, 0.01678, 0.0152, 0.00915, 0.00886, 0.00872, 0.00978, 0.01165, 0.00864, 0.01118, 0.01286, 0.00996, 0.0125, 0.01039, 0.01705, 0.00824, 0.00886, 0.00817, 0.00863, 0.0105, 0.00871, 0.08171, 0.01193, 0.01314, 0.01206, 0.01407, 0.01071, 0.01251, 0.01179, 0.01146, 0.00929, 0.01052, 0.01215, 0.0084, 0.00818, 0.00939, 0.0111, 0.00825, 0.01008, 0.01023, 0.00961, 0.0079, 0.01198, 0.0144, 0.00802, 0.01242, 0.00847, 0.01011, 0.00724, 0.00808, 0.0078, 0.00899, 0.00896, 0.00949, 0.00922, 0.01098, 0.01, 0.01342, 0.00965, 0.00844, 0.01778, 0.01504, 0.00876, 0.01126, 0.01156, 0.00994, 0.00745, 0.01045, 0.01139, 0.01102, 0.01004, 0.01044, 0.01421, 0.01363, 0.0147, 0.01748, 0.01497, 0.01481, 0.01661, 0.00933, 0.01088, 0.01211, 0.01187, 0.0114, 0.01087, 0.00985, 0.01082, 0.01058, 0.01129, 0.00882, 0.01084, 0.00902, 0.0079, 0.01036, 0.01589, 0.01561, 0.01591, 0.00899, 0.01108, 0.00841, 0.01003, 0.00851, 0.00882, 0.00846, 0.00785, 0.01152, 0.00747, 0.01326, 0.01202, 0.01211, 0.01078, 0.00952, 0.00873, 0.00881, 0.00874, 0.00915, 0.00875, 0.01297, 0.01552, 0.0151, 0.01016, 0.00992, 0.01251, 0.01115, 0.01149, 0.00982, 0.01462, 0.01529, 0.0145, 0.01056, 0.01488, 0.01365, 0.01448, 0.00917, 0.0134, 0.01205, 0.01572, 0.0126, 0.01488, 0.01305, 0.01335, 0.0138, 0.0164, 0.01209, 0.01237, 0.01442, 0.01402, 0.01277, 0.01318, 0.01188, 0.0129, 0.01144, 0.01322, 0.01297, 0.0121, 0.01209, 0.01029, 0.01079, 0.01249, 0.01233, 0.0121, 0.01022, 0.0128, 0.01174, 0.01218, 0.01303, 0.01323, 0.01318, 0.01287, 0.00961, 0.01202, 0.0124, 0.00992, 0.00876, 0.00935, 0.01319, 0.01636, 0.01632, 0.01494, 0.01298, 0.01614, 0.01406, 0.01537, 0.01153, 0.01115, 0.01271, 0.0107, 0.01222, 0.01248, 0.01198, 0.01383, 0.01146, 0.01187, 0.01068, 0.01125, 0.00998, 0.01224, 0.01454, 0.01162, 0.00956, 0.01122, 0.0154, 0.01199, 0.01342, 0.01294, 0.01456, 0.01293, 0.01589, 0.01161, 0.01349, 0.01587, 0.0161, 0.01506, 0.01604, 0.01245, 0.01415, 0.01038, 0.01375, 0.01225, 0.01179, 0.01138, 0.01149, 0.0114, 0.01157, 0.01201, 0.09678, 0.06875, 0.01665, 0.01943, 0.01672, 0.01779, 0.01975, 0.01513, 0.01188, 0.01383, 0.01055, 0.01209, 0.01624, 0.01171, 0.01034, 0.00943, 0.0124, 0.01104, 0.01002, 0.00883, 0.01064, 0.01032, 0.00949, 0.01005, 0.01087, 0.01209, 0.01055, 0.00979, 0.00997, 0.01044, 0.01106, 0.01088, 0.01076, 0.01045, 0.01152, 0.01085, 0.0105, 0.01114, 0.01146, 0.01082, 0.01229, 0.01175, 0.01162, 0.01101, 0.01116, 0.01256, 0.01128, 0.01152, 0.0107, 0.00988, 0.0095, 0.01009, 0.01045, 0.01003, 0.00992, 0.01213, 0.01087, 0.01368, 0.00953, 0.01064, 0.01243, 0.01214, 0.01155, 0.01008, 0.00976, 0.01033, 0.00912, 0.0081, 0.00967, 0.01116, 0.00911, 0.00921, 0.00997, 0.01136, 0.01025, 0.01241, 0.01273, 0.01327, 0.01109, 0.01279, 0.01226, 0.0121, 0.01061, 0.01401, 0.0134, 0.01432, 0.01133, 0.01394, 0.01414, 0.01459, 0.01155, 0.01481, 0.01262, 0.01169, 0.01079, 0.01328, 0.01375, 0.01229, 0.01428, 0.01132, 0.0128, 0.01126, 0.01216, 0.01314, 0.01251, 0.01231, 0.01489, 0.10504, 0.01146, 0.01181, 0.10182, 0.00974, 0.01066, 0.01245, 0.01188, 0.01268, 0.01247, 0.01243, 0.0136, 0.0116, 0.01212, 0.01459, 0.01641, 0.0161, 0.01189, 0.01301, 0.01594, 0.01101, 0.01209, 0.0146, 0.01388, 0.01439, 0.01206, 0.01364, 0.01212, 0.01313, 0.01581, 0.01511, 0.01362, 0.01411, 0.0139, 0.01423, 0.01307, 0.01509, 0.01644, 0.01567, 0.01653, 0.01601, 0.0161, 0.01324, 0.01587, 0.01735, 0.01691, 0.01574, 0.01699, 0.01222, 0.01273, 0.0119]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00124, 0.00087, 0.00088, 0.00087, 0.00086, 0.00085, 0.00085, 0.00085, 0.00098, 0.00088, 0.00087, 0.00087, 0.00087, 0.00088, 0.00085, 0.00085, 0.00086, 0.00082, 0.00084, 0.00083, 0.00103, 0.00352, 0.00085, 0.00084, 0.00084, 0.00089, 0.00086, 0.00084, 0.00085, 0.00084, 0.00085, 0.00087, 0.00085, 0.00085, 0.00086, 0.00086, 0.00084, 0.00086, 0.00086, 0.00085, 0.00087, 0.00086, 0.00085, 0.00087, 0.00084, 0.00086, 0.00085, 0.00084, 0.00167, 0.00083, 0.00086, 0.00111, 0.00108, 0.00101, 0.00084, 0.00085, 0.00085, 0.00086, 0.00084, 0.00084, 0.00086, 0.00083, 0.00083, 0.00083, 0.00111, 0.0009, 0.00086, 0.00088, 0.00086, 0.00084, 0.00086, 0.00084, 0.00091, 0.00085, 0.00084, 0.00087, 0.00083, 0.00083, 0.00241, 0.00085, 0.00086, 0.00109, 0.00086, 0.00085, 0.00085, 0.00084, 0.00084, 0.00084, 0.00084, 0.00092, 0.00087, 0.00083, 0.00087, 0.00532, 0.00083, 0.00085, 0.00101, 0.00113, 0.0011, 0.00089, 0.00088, 0.00086, 0.00113, 0.00084, 0.00122, 0.00087, 0.00086, 0.00085, 0.00086, 0.00088, 0.00085, 0.00088, 0.0031, 0.00085, 0.00087, 0.00085, 0.001, 0.00116, 0.00088, 0.00088, 0.00086, 0.00085, 0.00085, 0.00084, 0.00426, 0.00086, 0.00086, 0.00116, 0.00089, 0.00087, 0.00087, 0.00085, 0.00085, 0.00084, 0.00087, 0.00084, 0.00084, 0.0009, 0.00108, 0.00085, 0.00085, 0.00086, 0.00086, 0.00088, 0.00084, 0.00085, 0.00084, 0.00104, 0.00087, 0.00104, 0.00084, 0.00083, 0.00084, 0.00086, 0.00086, 0.00087, 0.00084, 0.00083, 0.00086, 0.00218, 0.00084, 0.004, 0.00086, 0.00087, 0.00087, 0.00105, 0.00103, 0.00103, 0.00107, 0.00089, 0.00107, 0.00114, 0.00113, 0.00085, 0.00107, 0.00086, 0.00089, 0.00088, 0.00089, 0.00086, 0.00085, 0.00085, 0.00086, 0.00088, 0.00087, 0.00085, 0.00086, 0.00087, 0.00085, 0.00085, 0.00087, 0.00089, 0.00085, 0.00088, 0.00087, 0.00086, 0.00241, 0.00085, 0.00084, 0.00087, 0.00099, 0.001, 0.00108, 0.00085, 0.00084, 0.00086, 0.00085, 0.00088, 0.00085, 0.00085, 0.00084, 0.00086, 0.00088, 0.00084, 0.00085, 0.00087, 0.00087, 0.00087, 0.00111, 0.00086, 0.00085, 0.00086, 0.00086, 0.00084, 0.00083, 0.00084, 0.00083, 0.00088, 0.00084, 0.00085, 0.0011, 0.0011, 0.00116, 0.00089, 0.00115, 0.00087, 0.00378, 0.00087, 0.00085, 0.00085, 0.0009, 0.00086, 0.00089, 0.00086, 0.00085, 0.00085, 0.00084, 0.00087, 0.00086, 0.00086, 0.00104, 0.00088, 0.00085, 0.00115, 0.00106, 0.00088, 0.00086, 0.00106, 0.00086, 0.00087, 0.00086, 0.0026, 0.00449, 0.00471, 0.00277, 0.00087, 0.00088, 0.00085, 0.00107, 0.0011, 0.00118, 0.00086, 0.00089, 0.00084, 0.00084, 0.00084, 0.00085, 0.00087, 0.00108, 0.0011, 0.00098, 0.00109, 0.00111, 0.0011, 0.0011, 0.0011, 0.0011, 0.00111, 0.00111, 0.00107, 0.0011, 0.00103, 0.00103, 0.00111, 0.00112, 0.00109, 0.00106, 0.00108, 0.00103, 0.00103, 0.00111, 0.00102, 0.00112, 0.00112, 0.00111, 0.00112, 0.00109, 0.00329, 0.00093, 0.00085, 0.00089, 0.00085, 0.00089, 0.00087, 0.00086, 0.00536, 0.0011, 0.00111, 0.00111, 0.00116, 0.00086, 0.00084, 0.00087, 0.0009, 0.00085, 0.00084, 0.00087, 0.00086, 0.00087, 0.00086, 0.00084, 0.00085, 0.00088, 0.00086, 0.00086, 0.00417, 0.00088, 0.00121, 0.00085, 0.00085, 0.00085, 0.00085, 0.00095, 0.00116, 0.00086, 0.00086, 0.00086, 0.00499, 0.00318, 0.00107, 0.00371, 0.00087, 0.00089, 0.00087, 0.00086, 0.00085, 0.00084, 0.00084, 0.00086, 0.00083, 0.00088, 0.00085, 0.00085, 0.00087, 0.00085, 0.00087, 0.00086, 0.00086, 0.00087, 0.00085, 0.00084, 0.00085, 0.00085, 0.00086, 0.00086, 0.00085, 0.00084, 0.00088, 0.00086, 0.00085, 0.00086, 0.00085, 0.0009, 0.00095, 0.00448, 0.00088, 0.00088, 0.00089, 0.00089, 0.00086, 0.00087, 0.00087, 0.0009, 0.00086, 0.00086, 0.00088, 0.00087, 0.00088, 0.0009, 0.00101]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00038, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00031, 0.00032, 0.00032, 0.00034, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00034, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00033, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00034, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00034, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00033, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00034, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00033, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00033, 0.00033, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00033, 0.00032, 0.00034, 0.00032, 0.00032, 0.00031, 0.00032, 0.00034, 0.00034, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.1656, 0.00059, 0.0006, 0.0006, 0.00059, 0.00062, 0.0006, 0.00059, 0.00058, 0.0006, 0.00059, 0.00058, 0.00059, 0.00059, 0.0006, 0.00058, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00065, 0.00064, 0.00063, 0.00059, 0.00059, 0.0006, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00061, 0.0006, 0.00058, 0.00064, 0.00058, 0.00058, 0.0006, 0.0006, 0.00058, 0.00058, 0.00059, 0.00059, 0.00063, 0.00059, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00063, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00064, 0.00058, 0.0006, 0.00059, 0.00059, 0.00059, 0.0006, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.0006, 0.00058, 0.0006, 0.00059, 0.0006, 0.0006, 0.00057, 0.00059, 0.0006, 0.00058, 0.00059, 0.00059, 0.00064, 0.00058, 0.00059, 0.00063, 0.00059, 0.00058, 0.00059, 0.0006, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00063, 0.00059, 0.00059, 0.00059, 0.00057, 0.00058, 0.00059, 0.00058, 0.00062, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.0006, 0.00058, 0.0006, 0.00058, 0.00062, 0.00059, 0.00063, 0.0006, 0.00059, 0.00059, 0.0006, 0.00059, 0.00059, 0.00059, 0.00058, 0.00063, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.0006, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00057, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.0006, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00059, 0.00059, 0.0006, 0.00063, 0.00059, 0.00059, 0.00058, 0.00059, 0.00062, 0.00062, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00059, 0.00074, 0.00059, 0.00059, 0.00059, 0.0006, 0.0006, 0.0006, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00064, 0.00059, 0.00063, 0.00059, 0.00059, 0.0006, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.0006, 0.0006, 0.00059, 0.00058, 0.00058, 0.00057, 0.00058, 0.0006, 0.00059, 0.00058, 0.00058, 0.00059, 0.0006, 0.00058, 0.00058, 0.00059, 0.00058, 0.00057, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.0006, 0.00058, 0.00065, 0.00059, 0.00062, 0.00058, 0.00057, 0.00061, 0.00059, 0.00059, 0.00058, 0.0006, 0.00063, 0.00059, 0.00058, 0.00059, 0.00058, 0.00062, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.0006, 0.0006, 0.00059, 0.00058, 0.00059, 0.0006, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00064, 0.00058, 0.00057, 0.00059, 0.00058, 0.00059, 0.00057, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00064, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00057, 0.00059, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00063, 0.00058, 0.00063, 0.00059, 0.0006, 0.00057, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00062, 0.00062, 0.00058, 0.00057, 0.00058, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.0006, 0.0006, 0.00058, 0.00058, 0.00059, 0.00063, 0.00057, 0.00059, 0.00063, 0.00059, 0.00059, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00059, 0.00063, 0.00059, 0.00059, 0.00059, 0.00059, 0.0006, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00059, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00016, 0.00012, 0.00011, 0.00011, 0.00011, 0.00011, 0.00012, 0.00011, 0.00012, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00012, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00012, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.0001, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00012, 0.00011, 0.00011, 0.00012, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00012, 0.00011, 0.00012, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00012, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.0001, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00012, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.00012, 0.00012, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00012, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.00012, 0.00011, 0.0001, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00012, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.00012, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.00012, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.00012, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.00019, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.0001, 0.00012, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.00011, 0.00011, 0.0001, 0.0001, 0.0001, 0.0001, 0.00011, 0.0001, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.00011, 0.0001, 0.00011, 0.00011, 0.00011]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.25848, 0.00058, 0.00058, 0.00057, 0.00057, 0.00058, 0.00058, 0.00057, 0.00057, 0.00058, 0.00057, 0.00057, 0.00056, 0.00056, 0.00057, 0.00056, 0.00059, 0.00056, 0.00056, 0.00055, 0.00056, 0.00057, 0.00056, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00057, 0.00056, 0.00057, 0.00056, 0.00055, 0.00055, 0.00057, 0.00057, 0.00058, 0.00055, 0.00056, 0.00056, 0.00056, 0.00055, 0.00057, 0.00056, 0.00056, 0.00056, 0.00058, 0.00056, 0.00055, 0.00056, 0.00056, 0.00057, 0.00057, 0.0006, 0.00056, 0.00056, 0.00055, 0.00056, 0.00056, 0.00055, 0.00056, 0.00055, 0.00055, 0.00056, 0.00057, 0.00057, 0.00056, 0.00056, 0.00056, 0.00057, 0.00059, 0.00056, 0.00058, 0.00056, 0.00056, 0.00057, 0.00055, 0.00055, 0.00056, 0.00056, 0.00056, 0.00071, 0.00056, 0.00056, 0.00057, 0.00057, 0.00055, 0.00056, 0.00055, 0.0006, 0.00055, 0.00056, 0.00055, 0.00055, 0.00057, 0.00055, 0.00055, 0.00057, 0.00046, 0.00057, 0.00057, 0.00057, 0.00056, 0.00055, 0.00071, 0.00056, 0.00056, 0.00057, 0.00057, 0.00047, 0.00056, 0.00048, 0.00046, 0.00056, 0.00057, 0.00055, 0.00055, 0.00056, 0.00055, 0.00057, 0.00056, 0.00056, 0.00056, 0.00056, 0.00046, 0.00056, 0.00055, 0.00055, 0.00056, 0.00058, 0.00045, 0.00056, 0.00057, 0.00055, 0.00057, 0.00055, 0.00055, 0.00055, 0.00056, 0.00056, 0.00055, 0.00055, 0.00057, 0.00046, 0.00046, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00055, 0.00055, 0.00055, 0.00056, 0.00057, 0.00055, 0.00055, 0.00057, 0.00057, 0.00064, 0.00056, 0.00056, 0.00057, 0.00057, 0.00055, 0.00056, 0.00055, 0.00055, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00055, 0.00056, 0.00055, 0.00055, 0.00056, 0.00057, 0.00056, 0.00057, 0.00056, 0.00055, 0.00058, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00056, 0.00056, 0.00055, 0.00056, 0.00056, 0.00055, 0.00057, 0.00077, 0.00056, 0.00056, 0.00057, 0.00056, 0.00057, 0.00056, 0.00057, 0.00056, 0.00056, 0.00055, 0.00056, 0.00058, 0.00055, 0.00056, 0.00055, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00055, 0.00054, 0.00055, 0.00055, 0.00056, 0.00062, 0.00058, 0.00055, 0.00056, 0.00056, 0.00056, 0.00056, 0.00055, 0.00055, 0.00055, 0.00056, 0.00056, 0.00055, 0.00057, 0.00057, 0.00056, 0.00055, 0.00055, 0.00055, 0.00055, 0.00058, 0.00055, 0.00056, 0.00056, 0.00056, 0.00055, 0.00056, 0.00055, 0.00056, 0.00056, 0.00057, 0.00057, 0.00061, 0.00057, 0.00057, 0.00056, 0.00057, 0.00055, 0.00056, 0.00056, 0.00056, 0.00058, 0.00056, 0.00057, 0.00055, 0.0006, 0.00056, 0.00057, 0.00056, 0.00055, 0.00056, 0.00056, 0.00057, 0.00057, 0.00055, 0.00056, 0.00056, 0.0006, 0.00063, 0.00057, 0.00056, 0.00056, 0.00057, 0.00058, 0.00056, 0.00059, 0.00057, 0.00056, 0.00055, 0.00056, 0.00064, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00057, 0.00068, 0.00056, 0.00056, 0.00056, 0.00058, 0.00056, 0.00059, 0.00056, 0.00055, 0.00057, 0.00057, 0.00055, 0.00057, 0.00056, 0.00057, 0.00057, 0.00056, 0.00056, 0.00055, 0.00057, 0.00057, 0.00055, 0.00056, 0.00057, 0.00056, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00058, 0.00056, 0.00055, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00076, 0.00058, 0.00057, 0.00057, 0.00056, 0.00056, 0.00057, 0.00056, 0.00056, 0.00056, 0.00057, 0.00057, 0.00056, 0.00055, 0.00055, 0.00057, 0.00056, 0.00056, 0.00056, 0.00055, 0.00056, 0.00057, 0.00056, 0.00055, 0.00061, 0.00056, 0.00055, 0.00056, 0.00055, 0.00056, 0.00056, 0.00055, 0.00057, 0.00055, 0.00055, 0.00056, 0.00057, 0.00056, 0.00057, 0.00056, 0.00056, 0.00056, 0.00057, 0.00057, 0.00057, 0.00057, 0.00057, 0.00057, 0.00057, 0.00056, 0.00056, 0.00056, 0.00056, 0.00056]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00381, 0.00273, 0.0027, 0.0027, 0.00273, 0.00271, 0.00267, 0.00283, 0.00274, 0.00269, 0.0027, 0.00269, 0.00272, 0.00273, 0.0027, 0.0027, 0.00269, 0.00268, 0.0027, 0.0027, 0.00273, 0.00272, 0.00268, 0.0027, 0.00278, 0.00278, 0.00271, 0.00269, 0.00268, 0.0027, 0.00271, 0.00271, 0.00269, 0.00273, 0.00271, 0.0027, 0.00267, 0.00269, 0.0027, 0.00271, 0.00271, 0.00269, 0.00269, 0.00267, 0.00269, 0.00269, 0.00269, 0.0027, 0.0027, 0.00271, 0.00271, 0.00288, 0.00277, 0.00297, 0.0027, 0.00269, 0.00268, 0.00269, 0.00268, 0.00269, 0.00269, 0.0027, 0.00268, 0.0027, 0.00272, 0.00269, 0.0027, 0.00271, 0.00273, 0.0027, 0.00284, 0.0027, 0.00271, 0.00282, 0.0027, 0.00268, 0.00268, 0.00268, 0.0027, 0.0027, 0.00272, 0.00496, 0.0027, 0.00268, 0.00269, 0.00269, 0.00271, 0.00269, 0.00271, 0.00292, 0.0027, 0.00269, 0.00269, 0.00268, 0.00269, 0.00271, 0.00271, 0.00275, 0.00271, 0.00271, 0.00268, 0.00271, 0.00291, 0.00269, 0.00286, 0.00271, 0.00269, 0.00269, 0.00271, 0.00269, 0.0027, 0.00272, 0.00269, 0.00267, 0.00268, 0.00269, 0.00272, 0.00269, 0.00272, 0.0027, 0.00268, 0.00268, 0.00269, 0.0027, 0.00269, 0.0027, 0.00272, 0.0027, 0.00271, 0.00269, 0.00273, 0.0027, 0.0027, 0.0027, 0.00268, 0.00269, 0.0027, 0.00272, 0.00271, 0.00271, 0.00269, 0.0027, 0.00267, 0.00271, 0.00269, 0.00268, 0.00268, 0.0027, 0.00269, 0.00269, 0.00267, 0.0027, 0.00268, 0.00269, 0.0027, 0.0027, 0.00269, 0.00269, 0.00268, 0.00269, 0.00269, 0.00269, 0.00269, 0.00281, 0.0028, 0.00273, 0.00272, 0.00273, 0.00273, 0.00274, 0.00271, 0.00272, 0.0027, 0.00271, 0.0027, 0.00271, 0.00273, 0.00271, 0.00269, 0.00271, 0.00272, 0.00272, 0.00272, 0.0027, 0.00269, 0.00281, 0.00272, 0.00282, 0.00271, 0.0027, 0.00269, 0.00272, 0.00273, 0.00271, 0.00269, 0.0027, 0.0027, 0.00269, 0.00271, 0.00271, 0.00282, 0.00271, 0.00269, 0.00271, 0.0027, 0.00313, 0.0027, 0.00269, 0.00271, 0.00271, 0.0027, 0.0027, 0.00271, 0.00269, 0.00278, 0.00269, 0.00272, 0.00278, 0.00271, 0.0027, 0.00269, 0.00271, 0.0027, 0.0027, 0.0027, 0.00269, 0.00271, 0.00271, 0.00269, 0.00272, 0.00271, 0.00296, 0.00271, 0.00271, 0.0027, 0.00271, 0.00271, 0.00275, 0.00269, 0.00267, 0.00271, 0.00274, 0.00267, 0.00271, 0.0027, 0.00273, 0.00272, 0.00271, 0.00271, 0.00273, 0.00272, 0.0027, 0.00274, 0.00273, 0.0027, 0.00272, 0.00271, 0.0027, 0.00271, 0.00265, 0.00264, 0.00264, 0.00273, 0.00262, 0.00291, 0.00266, 0.00273, 0.00265, 0.00265, 0.00263, 0.00265, 0.00264, 0.00274, 0.00272, 0.00262, 0.00274, 0.00265, 0.00273, 0.00264, 0.00274, 0.00264, 0.00274, 0.0028, 0.00265, 0.00263, 0.00263, 0.00272, 0.00271, 0.00276, 0.00267, 0.00265, 0.00262, 0.00272, 0.00277, 0.00264, 0.00269, 0.00264, 0.00264, 0.00272, 0.00271, 0.00294, 0.00388, 0.00268, 0.00273, 0.00273, 0.00265, 0.00357, 0.00265, 0.00304, 0.00272, 0.00261, 0.00268, 0.0027, 0.00266, 0.00267, 0.00264, 0.00278, 0.00274, 0.00267, 0.00269, 0.00268, 0.0027, 0.00269, 0.0027, 0.00269, 0.0027, 0.00271, 0.00269, 0.00267, 0.0027, 0.00268, 0.0027, 0.00272, 0.00271, 0.0027, 0.00272, 0.00272, 0.00274, 0.00269, 0.00313, 0.00269, 0.00269, 0.00269, 0.00271, 0.00271, 0.00273, 0.00283, 0.0027, 0.00269, 0.00278, 0.00276, 0.00271, 0.00271, 0.0027, 0.0027, 0.00271, 0.00272, 0.00271, 0.00272, 0.00271, 0.00271, 0.00268, 0.00273, 0.00271, 0.00269, 0.0027, 0.00273, 0.00275, 0.00269, 0.00273, 0.00271, 0.00271, 0.0027, 0.00272, 0.00269, 0.00269, 0.00272, 0.00274, 0.00271, 0.00272, 0.00272, 0.0027, 0.0027, 0.00272, 0.0027, 0.00271, 0.00271, 0.00273, 0.00271, 0.00268, 0.0027, 0.00271, 0.00273, 0.00272, 0.0027, 0.00269, 0.00272, 0.00272, 0.0027, 0.00271]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0026, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00044, 0.00051, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00046, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00044, 0.00044, 0.00045, 0.00046, 0.00045, 0.00044, 0.00044, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00046, 0.00045, 0.00045, 0.00048, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00046, 0.00045, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00044, 0.00057, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.0005, 0.00044, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00049, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00059, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00051, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00061, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00044, 0.00045, 0.00045, 0.00044, 0.00046, 0.00054, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00045, 0.00044, 0.00044, 0.00044, 0.00045, 0.00044, 0.00044, 0.00045, 0.00044, 0.00045, 0.00044, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00055, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00044, 0.00044, 0.00045, 0.00046, 0.00045, 0.00044, 0.00076, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00048, 0.00045, 0.00045, 0.00048, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00052, 0.0005, 0.00056, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00055, 0.00049, 0.0005, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00051, 0.00049, 0.00049, 0.00049, 0.00066, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00048, 0.00049, 0.0005, 0.00049, 0.00049, 0.00068, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00067, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00063, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00048, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00068, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00076, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00052, 0.00049, 0.00066, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00051, 0.0005, 0.0005, 0.00072, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.0005, 0.00049, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00052, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00066, 0.0005, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00052, 0.0005, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00053, 0.00049, 0.00052, 0.00049, 0.00049, 0.00049, 0.00076, 0.00049, 0.0005, 0.00049, 0.0005, 0.00049, 0.00064, 0.0005, 0.00051, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00066, 0.00049, 0.00051, 0.00063, 0.0005, 0.0005, 0.0005, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.00051, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.00053, 0.0005, 0.00073, 0.00072, 0.00072, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.0005, 0.00051, 0.00051, 0.0005, 0.00049, 0.0005, 0.0005, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.00049, 0.0005, 0.00049, 0.00049, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.00051, 0.0005, 0.0005, 0.0005, 0.00049, 0.0005]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.26785, 0.00472, 0.00469, 0.00468, 0.0047, 0.00469, 0.00466, 0.00479, 0.00473, 0.00465, 0.00467, 0.00466, 0.00467, 0.00467, 0.00464, 0.00466, 0.00468, 0.00461, 0.00465, 0.00464, 0.00469, 0.00469, 0.00464, 0.00465, 0.00473, 0.00473, 0.00467, 0.00463, 0.00464, 0.00465, 0.00468, 0.00467, 0.00464, 0.00516, 0.00466, 0.00468, 0.00465, 0.00465, 0.00465, 0.00469, 0.00466, 0.00464, 0.00465, 0.00462, 0.00463, 0.00466, 0.00466, 0.00464, 0.00465, 0.00466, 0.00468, 0.00483, 0.00473, 0.005, 0.00465, 0.00465, 0.00463, 0.00466, 0.00463, 0.00463, 0.00465, 0.00465, 0.00461, 0.00465, 0.00467, 0.00467, 0.00464, 0.00464, 0.00468, 0.00465, 0.00483, 0.00466, 0.0047, 0.00478, 0.00466, 0.00466, 0.00461, 0.00462, 0.00467, 0.00465, 0.00469, 0.00749, 0.00467, 0.00465, 0.00466, 0.00466, 0.00465, 0.00465, 0.00465, 0.00495, 0.00465, 0.00465, 0.00463, 0.00463, 0.00466, 0.00467, 0.00464, 0.00472, 0.00456, 0.00469, 0.00464, 0.00466, 0.0049, 0.00463, 0.00555, 0.00466, 0.00464, 0.00464, 0.00466, 0.00456, 0.00466, 0.0046, 0.00453, 0.00464, 0.00465, 0.00461, 0.00466, 0.00495, 0.00466, 0.00467, 0.00463, 0.00461, 0.00463, 0.00465, 0.00458, 0.00465, 0.00467, 0.00464, 0.00466, 0.00467, 0.00456, 0.00464, 0.00465, 0.00464, 0.00465, 0.00462, 0.00462, 0.00464, 0.00466, 0.00465, 0.00464, 0.00465, 0.00463, 0.00456, 0.00455, 0.00464, 0.00462, 0.00466, 0.00464, 0.00466, 0.00461, 0.00462, 0.00463, 0.00464, 0.00468, 0.00465, 0.00462, 0.00463, 0.00466, 0.00465, 0.00472, 0.00464, 0.00465, 0.00477, 0.00511, 0.00469, 0.00467, 0.00467, 0.00468, 0.00471, 0.00465, 0.00468, 0.00465, 0.00522, 0.00464, 0.00465, 0.00466, 0.00465, 0.00464, 0.00465, 0.00465, 0.00466, 0.00467, 0.00466, 0.00464, 0.00475, 0.00467, 0.0048, 0.00468, 0.00466, 0.00466, 0.00467, 0.00478, 0.00466, 0.00469, 0.00465, 0.00466, 0.00465, 0.00499, 0.0047, 0.00568, 0.00465, 0.00465, 0.00466, 0.00466, 0.00541, 0.00464, 0.00465, 0.00465, 0.00465, 0.00463, 0.00465, 0.00469, 0.00464, 0.00473, 0.00463, 0.00466, 0.00474, 0.00466, 0.00465, 0.00464, 0.00467, 0.00464, 0.00466, 0.00464, 0.00462, 0.00464, 0.00466, 0.00463, 0.00467, 0.00467, 0.00542, 0.00468, 0.00466, 0.00465, 0.00465, 0.00467, 0.0047, 0.00463, 0.00461, 0.00466, 0.00468, 0.00464, 0.00466, 0.00467, 0.00468, 0.00467, 0.00465, 0.00467, 0.00468, 0.00465, 0.00469, 0.00468, 0.00468, 0.00464, 0.00466, 0.00467, 0.00464, 0.00464, 0.00461, 0.00462, 0.00463, 0.0047, 0.00464, 0.00489, 0.00464, 0.00469, 0.0046, 0.00459, 0.00459, 0.0046, 0.00459, 0.00472, 0.00501, 0.00458, 0.00468, 0.00465, 0.00469, 0.00461, 0.00469, 0.00458, 0.0047, 0.00478, 0.0046, 0.00464, 0.00461, 0.00468, 0.00468, 0.00476, 0.00469, 0.00461, 0.00457, 0.00469, 0.00472, 0.00468, 0.00464, 0.00467, 0.00461, 0.00467, 0.00463, 0.00558, 0.00601, 0.00464, 0.0047, 0.0047, 0.00459, 0.00574, 0.00463, 0.00519, 0.00467, 0.00462, 0.00464, 0.00469, 0.00461, 0.00476, 0.00462, 0.00501, 0.00471, 0.00465, 0.0049, 0.00465, 0.00465, 0.00465, 0.00465, 0.00462, 0.00466, 0.00466, 0.00465, 0.00463, 0.00464, 0.00464, 0.00465, 0.00468, 0.00466, 0.00465, 0.00469, 0.00468, 0.0047, 0.00466, 0.00514, 0.00464, 0.00465, 0.00469, 0.00468, 0.00511, 0.00511, 0.00571, 0.00469, 0.00467, 0.00473, 0.00471, 0.00465, 0.00469, 0.00466, 0.00464, 0.00465, 0.00468, 0.00467, 0.00468, 0.00465, 0.00464, 0.00464, 0.00468, 0.00467, 0.00464, 0.00464, 0.00467, 0.00472, 0.00466, 0.00466, 0.00473, 0.00466, 0.00465, 0.00468, 0.00463, 0.00465, 0.00465, 0.00469, 0.00467, 0.00465, 0.00469, 0.00464, 0.00467, 0.00468, 0.00468, 0.00467, 0.00468, 0.00469, 0.00467, 0.00465, 0.00466, 0.00468, 0.0047, 0.0047, 0.00469, 0.00467, 0.00475, 0.00469, 0.00466, 0.00467]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84303, 10.87155, 10.85032, 10.81087, 10.64537, 10.63943, 10.42704, 10.13551, 9.93496, 9.83494, 9.58592, 9.84757, 9.88552, 9.63097, 9.79022, 9.51147, 9.4606, 9.65582, 9.39007, 9.33886, 9.24978, 9.152, 9.18226, 9.00447, 9.19856, 9.06681, 9.16059, 9.16939, 9.30049, 8.98819, 8.92948, 9.0507, 9.0463, 8.66041, 8.72526, 8.75716, 8.69559, 8.74303, 8.66681, 8.77472, 8.67057, 8.8619, 8.84447, 8.50989, 8.39988, 8.43941, 8.49864, 8.39575, 8.4422, 8.59464, 8.37842, 8.20138, 8.236, 8.2319, 8.27672, 7.92273, 8.10152, 7.8984, 8.25217, 8.23541, 8.01089, 7.97596, 7.92706, 7.74403, 7.7485, 7.65015, 7.52079, 7.9112, 7.70347, 7.45605, 7.74759, 7.77568, 7.54533, 7.30357, 7.45723, 7.3426, 7.46645, 7.22831, 7.63649, 7.28211, 7.34866, 7.21221, 7.21132, 7.41795, 7.17177, 7.28168, 6.99581, 7.004, 7.04074, 7.1367, 6.82354, 6.98508, 7.08921, 6.99769, 6.87461, 6.75657, 6.99031, 7.05959, 6.70411, 6.5827, 6.72604, 6.74348, 6.73218, 6.73708, 6.65685, 6.4055, 6.63559, 6.61892, 6.44639, 6.62609, 6.74333, 6.61179, 6.7261, 6.69431, 6.62741, 6.50922, 6.59901, 6.40739, 6.6657, 6.24852, 6.25199, 6.30265, 6.39086, 6.34866, 6.4484, 6.29117, 6.33917, 6.23682, 6.20019, 6.39713, 6.32382, 6.32063, 6.16132, 6.15692, 6.23736, 6.38207, 6.20216, 6.14927, 6.18286, 6.11574, 6.06273, 6.07513, 6.25658, 6.40785, 6.25681, 6.2924, 6.09673, 6.17564, 6.00002, 6.02568, 5.95394, 6.24995, 6.18499, 5.96441, 5.78379, 6.12452, 5.8475, 6.10173, 5.78491, 6.16542, 6.14406, 6.08134, 5.92727, 6.11254, 5.94363, 6.20077, 5.89399, 5.7901, 5.78128, 5.68813, 6.01482, 5.99528, 6.06741, 5.89085, 6.03981, 5.96811, 5.99655, 5.98984, 5.94628, 5.83848, 5.9481, 5.61614, 5.7002, 5.88656, 5.83806, 5.86311, 5.75859, 5.83316, 5.72072, 5.55659, 5.71965, 5.61978, 5.82718, 5.59717, 5.70318, 5.70327, 5.89853, 5.63883, 5.84367, 5.73571, 5.86365, 5.32462, 5.89684, 5.87059, 5.85018, 5.40966, 5.40521, 5.6244, 5.59463, 5.48385, 5.57514, 5.67111, 5.47486, 5.74063, 5.50617, 5.58954, 5.62055, 5.61722, 5.51063, 5.6138, 5.67042, 5.67814, 5.58421, 5.65728, 5.36779, 5.67697, 5.62608, 5.41953, 5.57893, 5.62664, 5.55034, 5.33858, 5.53624, 5.48821, 5.48891, 5.37489, 5.5499, 5.60024, 5.39139, 5.51868, 5.4935, 5.33216, 5.50746, 5.41318, 5.44698, 5.31869, 5.06634, 5.48126, 5.57099, 5.71639, 5.41515, 5.60293, 5.63581, 5.23321, 5.27358, 5.3934, 5.40049, 5.32861, 5.49563, 5.18115, 5.29818, 5.24632, 5.377, 5.25164, 5.44247, 5.53356, 5.31175, 5.43649, 5.33683, 5.07482, 5.31199, 5.25123, 5.30045, 5.10952, 5.27365, 5.26615, 5.4733, 5.15569, 5.2676, 5.21227, 5.35586, 4.98451, 4.91017, 5.32431, 5.38997, 5.22667, 5.3209, 5.10232, 5.16141, 5.26239, 5.0658, 5.26091, 5.06389, 5.34895, 5.24827, 5.1463, 5.24113, 5.03942, 5.31795, 5.05285, 5.02784, 5.14139, 5.11164, 5.27303, 5.15115, 5.2757, 5.09401, 5.09338, 5.24504, 5.32369, 5.25347, 5.19226, 5.14165, 5.29079, 4.95338, 5.20578, 5.09105, 5.30122, 5.17357, 5.19235, 5.11365, 4.98113, 4.9916, 5.22149, 5.30937, 5.10092, 5.0529, 4.91086, 5.12305, 5.11531, 4.92812, 5.3389, 5.02814, 5.10063, 5.16722, 5.00342, 5.0656, 5.06853, 5.0, 5.08165, 5.16456, 4.98252, 5.1839, 4.93148, 4.92569, 5.06682, 4.99595, 4.90624, 4.77517, 4.94606, 5.11508, 5.01539, 5.01397, 5.3327, 4.96029, 4.9915, 5.04439, 4.80654, 4.73199, 4.99639, 5.04237, 4.8734, 4.95425, 5.04678, 5.02392, 4.81994, 4.89463, 4.90711, 4.83288, 4.74257, 5.01934, 4.75352, 5.20696, 4.79359, 4.99212, 4.73894, 4.7885, 4.82299, 4.65617, 4.65522, 4.84524, 4.81217, 4.79792, 4.92038, 4.88607, 4.92565, 4.7712, 4.88216, 4.73528, 4.92078, 4.96145, 4.87447, 4.71317, 4.78702, 4.90462, 4.71624, 4.86657, 4.69712, 4.69196, 4.64876]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84303, 10.87155, 10.85032, 10.81087, 10.64537, 10.63943, 10.42704, 10.13551, 9.93496, 9.83494, 9.58592, 9.84757, 9.88552, 9.63097, 9.79022, 9.51147, 9.4606, 9.65582, 9.39007, 9.33886, 9.24978, 9.152, 9.18226, 9.00447, 9.19856, 9.06681, 9.16059, 9.16939, 9.30049, 8.98819, 8.92948, 9.0507, 9.0463, 8.66041, 8.72526, 8.75716, 8.69559, 8.74303, 8.66681, 8.77472, 8.67057, 8.8619, 8.84447, 8.50989, 8.39988, 8.43941, 8.49864, 8.39575, 8.4422, 8.59464, 8.37842, 8.20138, 8.236, 8.2319, 8.27672, 7.92273, 8.10152, 7.8984, 8.25217, 8.23541, 8.01089, 7.97596, 7.92706, 7.74403, 7.7485, 7.65015, 7.52079, 7.9112, 7.70347, 7.45605, 7.74759, 7.77568, 7.54533, 7.30357, 7.45723, 7.3426, 7.46645, 7.22831, 7.63649, 7.28211, 7.34866, 7.21221, 7.21132, 7.41795, 7.17177, 7.28168, 6.99581, 7.004, 7.04074, 7.1367, 6.82354, 6.98508, 7.08921, 6.99769, 6.87461, 6.75657, 6.99031, 7.05959, 6.70411, 6.5827, 6.72604, 6.74348, 6.73218, 6.73708, 6.65685, 6.4055, 6.63559, 6.61892, 6.44639, 6.62609, 6.74333, 6.61179, 6.7261, 6.69431, 6.62741, 6.50922, 6.59901, 6.40739, 6.6657, 6.24852, 6.25199, 6.30265, 6.39086, 6.34866, 6.4484, 6.29117, 6.33917, 6.23682, 6.20019, 6.39713, 6.32382, 6.32063, 6.16132, 6.15692, 6.23736, 6.38207, 6.20216, 6.14927, 6.18286, 6.11574, 6.06273, 6.07513, 6.25658, 6.40785, 6.25681, 6.2924, 6.09673, 6.17564, 6.00002, 6.02568, 5.95394, 6.24995, 6.18499, 5.96441, 5.78379, 6.12452, 5.8475, 6.10173, 5.78491, 6.16542, 6.14406, 6.08134, 5.92727, 6.11254, 5.94363, 6.20077, 5.89399, 5.7901, 5.78128, 5.68813, 6.01482, 5.99528, 6.06741, 5.89085, 6.03981, 5.96811, 5.99655, 5.98984, 5.94628, 5.83848, 5.9481, 5.61614, 5.7002, 5.88656, 5.83806, 5.86311, 5.75859, 5.83316, 5.72072, 5.55659, 5.71965, 5.61978, 5.82718, 5.59717, 5.70318, 5.70327, 5.89853, 5.63883, 5.84367, 5.73571, 5.86365, 5.32462, 5.89684, 5.87059, 5.85018, 5.40966, 5.40521, 5.6244, 5.59463, 5.48385, 5.57514, 5.67111, 5.47486, 5.74063, 5.50617, 5.58954, 5.62055, 5.61722, 5.51063, 5.6138, 5.67042, 5.67814, 5.58421, 5.65728, 5.36779, 5.67697, 5.62608, 5.41953, 5.57893, 5.62664, 5.55034, 5.33858, 5.53624, 5.48821, 5.48891, 5.37489, 5.5499, 5.60024, 5.39139, 5.51868, 5.4935, 5.33216, 5.50746, 5.41318, 5.44698, 5.31869, 5.06634, 5.48126, 5.57099, 5.71639, 5.41515, 5.60293, 5.63581, 5.23321, 5.27358, 5.3934, 5.40049, 5.32861, 5.49563, 5.18115, 5.29818, 5.24632, 5.377, 5.25164, 5.44247, 5.53356, 5.31175, 5.43649, 5.33683, 5.07482, 5.31199, 5.25123, 5.30045, 5.10952, 5.27365, 5.26615, 5.4733, 5.15569, 5.2676, 5.21227, 5.35586, 4.98451, 4.91017, 5.32431, 5.38997, 5.22667, 5.3209, 5.10232, 5.16141, 5.26239, 5.0658, 5.26091, 5.06389, 5.34895, 5.24827, 5.1463, 5.24113, 5.03942, 5.31795, 5.05285, 5.02784, 5.14139, 5.11164, 5.27303, 5.15115, 5.2757, 5.09401, 5.09338, 5.24504, 5.32369, 5.25347, 5.19226, 5.14165, 5.29079, 4.95338, 5.20578, 5.09105, 5.30122, 5.17357, 5.19235, 5.11365, 4.98113, 4.9916, 5.22149, 5.30937, 5.10092, 5.0529, 4.91086, 5.12305, 5.11531, 4.92812, 5.3389, 5.02814, 5.10063, 5.16722, 5.00342, 5.0656, 5.06853, 5.0, 5.08165, 5.16456, 4.98252, 5.1839, 4.93148, 4.92569, 5.06682, 4.99595, 4.90624, 4.77517, 4.94606, 5.11508, 5.01539, 5.01397, 5.3327, 4.96029, 4.9915, 5.04439, 4.80654, 4.73199, 4.99639, 5.04237, 4.8734, 4.95425, 5.04678, 5.02392, 4.81994, 4.89463, 4.90711, 4.83288, 4.74257, 5.01934, 4.75352, 5.20696, 4.79359, 4.99212, 4.73894, 4.7885, 4.82299, 4.65617, 4.65522, 4.84524, 4.81217, 4.79792, 4.92038, 4.88607, 4.92565, 4.7712, 4.88216, 4.73528, 4.92078, 4.96145, 4.87447, 4.71317, 4.78702, 4.90462, 4.71624, 4.86657, 4.69712, 4.69196, 4.64876]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.92196, 13.29306, 13.8377, 12.64037, 11.97375, 9.45262, 6.78823, 6.89004, 5.94557, 4.54615, 4.13637, 2.82375, 2.38927, 2.34389, 2.05973, 2.22596, 2.14457, 1.88597, 2.17986, 2.06069, 2.12423, 2.1677, 2.0115, 2.21442, 1.98307, 2.0966, 1.90389, 1.86829, 1.92477, 2.13027, 2.09469, 2.11211, 1.95723, 2.18758, 2.38519, 2.04808, 2.04244, 1.85027, 1.9837, 1.78603, 2.12943, 1.83753, 1.73653, 1.84787, 1.96175, 1.78052, 1.76095, 1.7401, 1.76961, 1.54057, 1.76088, 1.7938, 1.76365, 1.83855, 1.58517, 1.79545, 1.7158, 1.81815, 1.53518, 1.48648, 1.68949, 1.4562, 1.8648, 1.85145, 1.61928, 1.6745, 1.65487, 1.55646, 1.47797, 1.6989, 1.43883, 1.43836, 1.46011, 1.39711, 1.37457, 1.48663, 1.40785, 1.35385, 1.34051, 1.27757, 1.35283, 1.29709, 1.2816, 1.30185, 1.24092, 1.29738, 1.41961, 1.34489, 1.44199, 1.06928, 1.09491, 1.16108, 1.14396, 1.33634, 1.03654, 1.30756, 1.08982, 1.27845, 0.98191, 1.37412, 1.30793, 1.21672, 1.05131, 1.25909, 1.09643, 1.13996, 1.20961, 1.09191, 1.24074, 0.97878, 1.18535, 0.97714, 0.95456, 1.10186, 1.24389, 1.07847, 1.01822, 1.2519, 1.18392, 1.42087, 1.00253, 1.23223, 1.05494, 1.02956, 0.95692, 1.27887, 1.54081, 1.2168, 1.18019, 1.34805, 0.93443, 1.06987, 1.00938, 1.19729, 1.32572, 1.18029, 1.39724, 1.01719, 1.76109, 1.21222, 1.26256, 1.31969, 1.1555, 0.93801, 0.99546, 1.01521, 1.36553, 1.55577, 1.11391, 1.2491, 1.45721, 1.65042, 1.60593, 1.30243, 1.29342, 2.04924, 1.3376, 1.21234, 1.37945, 1.79037, 1.23389, 1.08215, 1.31811, 1.12901, 1.35786, 1.8341, 1.46143, 1.31586, 1.39491, 1.24546, 1.26969, 1.25412, 1.27022, 1.43967, 1.14847, 1.3362, 1.91114, 1.35642, 1.06973, 1.20518, 1.11732, 1.73877, 1.36915, 1.34679, 1.25766, 1.64809, 1.37397, 1.17279, 1.169, 1.49772, 1.11509, 1.29145, 1.479, 1.60514, 1.12787, 1.20465, 1.52478, 1.37769, 1.40825, 1.40433, 1.19434, 1.52129, 1.49087, 1.60752, 1.51416, 1.37753, 1.49097, 1.59106, 1.33146, 1.56964, 1.54958, 1.2024, 1.29844, 1.28184, 1.63096, 1.29563, 1.41842, 1.57651, 1.29669, 1.23902, 1.51872, 1.34276, 1.28172, 1.67239, 1.39643, 1.57361, 1.69097, 1.37206, 1.81716, 1.3501, 1.2879, 1.45938, 1.9477, 1.77504, 2.56828, 1.55284, 1.34454, 1.21685, 1.65336, 1.29693, 2.2136, 1.28644, 1.78502, 1.52285, 1.47963, 1.65183, 1.23421, 1.41797, 1.5183, 1.31219, 1.29375, 1.3932, 1.5544, 1.2678, 1.61107, 1.43809, 1.9371, 1.64335, 1.38939, 1.24473, 1.15131, 1.26598, 1.37433, 1.20588, 1.22283, 1.31678, 1.40086, 1.53213, 1.35367, 1.43407, 1.41639, 1.25063, 1.37444, 1.20928, 1.40445, 1.48011, 1.49606, 1.43456, 1.4511, 1.51505, 1.49329, 1.32736, 1.34283, 1.56947, 1.3986, 1.38533, 1.4325, 1.36846, 1.40113, 1.40195, 1.41944, 1.73207, 1.35246, 1.98477, 1.75001, 1.59412, 1.33312, 1.55175, 1.45641, 1.40103, 1.32697, 1.19674, 1.19056, 1.56111, 1.64, 1.52329, 1.62982, 1.42489, 1.1143, 1.42326, 1.36052, 1.20749, 1.49372, 1.38211, 1.6856, 1.48198, 1.34985, 1.48241, 1.24509, 1.40355, 1.44024, 1.31152, 1.30253, 1.59307, 1.35212, 1.78683, 1.61562, 1.61575, 1.46207, 1.29047, 1.55842, 1.39097, 1.35377, 1.50655, 1.67836, 1.37929, 1.32311, 1.35305, 1.77455, 1.48895, 1.40827, 1.23883, 1.35995, 1.46576, 1.39021, 1.55027, 1.27874, 1.53316, 1.30645, 1.32818, 1.41856, 1.40297, 1.19176, 1.73797, 1.28462, 1.46556, 1.31822, 1.27157, 1.29905, 1.43641, 1.37732, 1.32041, 1.45048, 1.30403, 1.12439, 1.41266, 1.49642, 1.41634, 1.48283, 1.73467, 1.90209, 1.41005, 1.66166, 1.51488, 1.35734, 1.47652, 1.40564, 1.6499, 1.41346, 1.24965, 1.34929, 1.35141, 1.18107, 1.30851, 1.17223, 1.29341, 1.38306, 1.247, 1.29013, 1.70946, 1.36584, 1.4061, 1.82813, 1.27073, 1.45088, 1.55944, 1.5925, 1.64727, 1.42815, 1.19955]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.92196, 13.29306, 13.8377, 12.64037, 11.97375, 9.45262, 6.78823, 6.89004, 5.94557, 4.54615, 4.13637, 2.82375, 2.38927, 2.34389, 2.05973, 2.22596, 2.14457, 1.88597, 2.17986, 2.06069, 2.12423, 2.1677, 2.0115, 2.21442, 1.98307, 2.0966, 1.90389, 1.86829, 1.92477, 2.13027, 2.09469, 2.11211, 1.95723, 2.18758, 2.38519, 2.04808, 2.04244, 1.85027, 1.9837, 1.78603, 2.12943, 1.83753, 1.73653, 1.84787, 1.96175, 1.78052, 1.76095, 1.7401, 1.76961, 1.54057, 1.76088, 1.7938, 1.76365, 1.83855, 1.58517, 1.79545, 1.7158, 1.81815, 1.53518, 1.48648, 1.68949, 1.4562, 1.8648, 1.85145, 1.61928, 1.6745, 1.65487, 1.55646, 1.47797, 1.6989, 1.43883, 1.43836, 1.46011, 1.39711, 1.37457, 1.48663, 1.40785, 1.35385, 1.34051, 1.27757, 1.35283, 1.29709, 1.2816, 1.30185, 1.24092, 1.29738, 1.41961, 1.34489, 1.44199, 1.06928, 1.09491, 1.16108, 1.14396, 1.33634, 1.03654, 1.30756, 1.08982, 1.27845, 0.98191, 1.37412, 1.30793, 1.21672, 1.05131, 1.25909, 1.09643, 1.13996, 1.20961, 1.09191, 1.24074, 0.97878, 1.18535, 0.97714, 0.95456, 1.10186, 1.24389, 1.07847, 1.01822, 1.2519, 1.18392, 1.42087, 1.00253, 1.23223, 1.05494, 1.02956, 0.95692, 1.27887, 1.54081, 1.2168, 1.18019, 1.34805, 0.93443, 1.06987, 1.00938, 1.19729, 1.32572, 1.18029, 1.39724, 1.01719, 1.76109, 1.21222, 1.26256, 1.31969, 1.1555, 0.93801, 0.99546, 1.01521, 1.36553, 1.55577, 1.11391, 1.2491, 1.45721, 1.65042, 1.60593, 1.30243, 1.29342, 2.04924, 1.3376, 1.21234, 1.37945, 1.79037, 1.23389, 1.08215, 1.31811, 1.12901, 1.35786, 1.8341, 1.46143, 1.31586, 1.39491, 1.24546, 1.26969, 1.25412, 1.27022, 1.43967, 1.14847, 1.3362, 1.91114, 1.35642, 1.06973, 1.20518, 1.11732, 1.73877, 1.36915, 1.34679, 1.25766, 1.64809, 1.37397, 1.17279, 1.169, 1.49772, 1.11509, 1.29145, 1.479, 1.60514, 1.12787, 1.20465, 1.52478, 1.37769, 1.40825, 1.40433, 1.19434, 1.52129, 1.49087, 1.60752, 1.51416, 1.37753, 1.49097, 1.59106, 1.33146, 1.56964, 1.54958, 1.2024, 1.29844, 1.28184, 1.63096, 1.29563, 1.41842, 1.57651, 1.29669, 1.23902, 1.51872, 1.34276, 1.28172, 1.67239, 1.39643, 1.57361, 1.69097, 1.37206, 1.81716, 1.3501, 1.2879, 1.45938, 1.9477, 1.77504, 2.56828, 1.55284, 1.34454, 1.21685, 1.65336, 1.29693, 2.2136, 1.28644, 1.78502, 1.52285, 1.47963, 1.65183, 1.23421, 1.41797, 1.5183, 1.31219, 1.29375, 1.3932, 1.5544, 1.2678, 1.61107, 1.43809, 1.9371, 1.64335, 1.38939, 1.24473, 1.15131, 1.26598, 1.37433, 1.20588, 1.22283, 1.31678, 1.40086, 1.53213, 1.35367, 1.43407, 1.41639, 1.25063, 1.37444, 1.20928, 1.40445, 1.48011, 1.49606, 1.43456, 1.4511, 1.51505, 1.49329, 1.32736, 1.34283, 1.56947, 1.3986, 1.38533, 1.4325, 1.36846, 1.40113, 1.40195, 1.41944, 1.73207, 1.35246, 1.98477, 1.75001, 1.59412, 1.33312, 1.55175, 1.45641, 1.40103, 1.32697, 1.19674, 1.19056, 1.56111, 1.64, 1.52329, 1.62982, 1.42489, 1.1143, 1.42326, 1.36052, 1.20749, 1.49372, 1.38211, 1.6856, 1.48198, 1.34985, 1.48241, 1.24509, 1.40355, 1.44024, 1.31152, 1.30253, 1.59307, 1.35212, 1.78683, 1.61562, 1.61575, 1.46207, 1.29047, 1.55842, 1.39097, 1.35377, 1.50655, 1.67836, 1.37929, 1.32311, 1.35305, 1.77455, 1.48895, 1.40827, 1.23883, 1.35995, 1.46576, 1.39021, 1.55027, 1.27874, 1.53316, 1.30645, 1.32818, 1.41856, 1.40297, 1.19176, 1.73797, 1.28462, 1.46556, 1.31822, 1.27157, 1.29905, 1.43641, 1.37732, 1.32041, 1.45048, 1.30403, 1.12439, 1.41266, 1.49642, 1.41634, 1.48283, 1.73467, 1.90209, 1.41005, 1.66166, 1.51488, 1.35734, 1.47652, 1.40564, 1.6499, 1.41346, 1.24965, 1.34929, 1.35141, 1.18107, 1.30851, 1.17223, 1.29341, 1.38306, 1.247, 1.29013, 1.70946, 1.36584, 1.4061, 1.82813, 1.27073, 1.45088, 1.55944, 1.5925, 1.64727, 1.42815, 1.19955]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [65.0, 80.0, 81.0, 75.0, 72.0, 103.0, 108.0, 112.0, 107.0, 122.0, 99.0, 159.0, 148.0, 150.0, 167.0, 157.0, 165.0, 144.0, 182.0, 187.0, 180.0, 162.0, 181.0, 129.0, 189.0, 148.0, 195.0, 190.0, 137.0, 181.0, 151.0, 155.0, 152.0, 166.0, 152.0, 170.0, 160.0, 209.0, 168.0, 214.0, 166.0, 181.0, 190.0, 185.0, 161.0, 162.0, 169.0, 187.0, 184.0, 239.0, 225.0, 187.0, 190.0, 131.0, 187.0, 182.0, 159.0, 161.0, 248.0, 226.0, 201.0, 211.0, 174.0, 164.0, 168.0, 225.0, 202.0, 174.0, 223.0, 202.0, 243.0, 235.0, 180.0, 239.0, 219.0, 205.0, 210.0, 192.0, 216.0, 207.0, 209.0, 245.0, 217.0, 227.0, 212.0, 207.0, 191.0, 173.0, 196.0, 193.0, 194.0, 186.0, 203.0, 189.0, 210.0, 160.0, 204.0, 187.0, 189.0, 159.0, 168.0, 209.0, 181.0, 159.0, 173.0, 153.0, 175.0, 152.0, 147.0, 174.0, 180.0, 153.0, 176.0, 146.0, 165.0, 154.0, 147.0, 106.0, 147.0, 133.0, 174.0, 148.0, 152.0, 143.0, 173.0, 127.0, 116.0, 130.0, 127.0, 123.0, 143.0, 142.0, 146.0, 123.0, 131.0, 124.0, 138.0, 139.0, 109.0, 107.0, 130.0, 103.0, 121.0, 157.0, 131.0, 148.0, 139.0, 96.0, 120.0, 101.0, 96.0, 102.0, 102.0, 122.0, 105.0, 84.0, 114.0, 117.0, 95.0, 90.0, 106.0, 137.0, 136.0, 131.0, 122.0, 95.0, 111.0, 99.0, 117.0, 119.0, 129.0, 111.0, 104.0, 112.0, 108.0, 102.0, 88.0, 97.0, 120.0, 121.0, 124.0, 96.0, 126.0, 134.0, 122.0, 98.0, 97.0, 115.0, 102.0, 102.0, 128.0, 120.0, 104.0, 104.0, 97.0, 112.0, 104.0, 96.0, 117.0, 97.0, 136.0, 100.0, 92.0, 104.0, 95.0, 111.0, 97.0, 87.0, 108.0, 128.0, 94.0, 111.0, 106.0, 122.0, 99.0, 94.0, 110.0, 104.0, 116.0, 119.0, 114.0, 112.0, 104.0, 104.0, 108.0, 88.0, 105.0, 114.0, 103.0, 105.0, 96.0, 98.0, 92.0, 92.0, 91.0, 102.0, 119.0, 106.0, 86.0, 104.0, 60.0, 110.0, 92.0, 91.0, 80.0, 91.0, 114.0, 106.0, 80.0, 119.0, 117.0, 112.0, 114.0, 98.0, 102.0, 109.0, 101.0, 100.0, 102.0, 126.0, 124.0, 99.0, 112.0, 110.0, 129.0, 111.0, 99.0, 119.0, 101.0, 82.0, 110.0, 84.0, 95.0, 104.0, 96.0, 107.0, 83.0, 114.0, 105.0, 93.0, 104.0, 108.0, 94.0, 99.0, 104.0, 101.0, 88.0, 112.0, 101.0, 101.0, 108.0, 119.0, 118.0, 103.0, 100.0, 107.0, 94.0, 104.0, 118.0, 111.0, 115.0, 100.0, 114.0, 90.0, 110.0, 107.0, 90.0, 91.0, 145.0, 113.0, 112.0, 120.0, 101.0, 98.0, 97.0, 96.0, 109.0, 100.0, 115.0, 120.0, 120.0, 121.0, 128.0, 103.0, 94.0, 104.0, 110.0, 89.0, 102.0, 106.0, 113.0, 117.0, 113.0, 115.0, 93.0, 114.0, 119.0, 132.0, 82.0, 112.0, 105.0, 96.0, 124.0, 107.0, 108.0, 104.0, 145.0, 119.0, 124.0, 115.0, 116.0, 94.0, 130.0, 98.0, 115.0, 117.0, 120.0, 122.0, 122.0, 110.0, 108.0, 87.0, 117.0, 102.0, 123.0, 108.0, 123.0, 107.0, 99.0, 127.0, 94.0, 107.0, 72.0, 102.0, 86.0, 91.0, 94.0, 116.0, 106.0, 120.0, 127.0, 115.0, 124.0, 126.0, 129.0, 117.0, 112.0, 120.0, 119.0, 126.0, 111.0, 119.0, 91.0, 102.0, 95.0, 118.0, 111.0, 99.0, 122.0, 125.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [65.0, 80.0, 81.0, 75.0, 72.0, 103.0, 108.0, 112.0, 107.0, 122.0, 99.0, 159.0, 148.0, 150.0, 167.0, 157.0, 165.0, 144.0, 182.0, 187.0, 180.0, 162.0, 181.0, 129.0, 189.0, 148.0, 195.0, 190.0, 137.0, 181.0, 151.0, 155.0, 152.0, 166.0, 152.0, 170.0, 160.0, 209.0, 168.0, 214.0, 166.0, 181.0, 190.0, 185.0, 161.0, 162.0, 169.0, 187.0, 184.0, 239.0, 225.0, 187.0, 190.0, 131.0, 187.0, 182.0, 159.0, 161.0, 248.0, 226.0, 201.0, 211.0, 174.0, 164.0, 168.0, 225.0, 202.0, 174.0, 223.0, 202.0, 243.0, 235.0, 180.0, 239.0, 219.0, 205.0, 210.0, 192.0, 216.0, 207.0, 209.0, 245.0, 217.0, 227.0, 212.0, 207.0, 191.0, 173.0, 196.0, 193.0, 194.0, 186.0, 203.0, 189.0, 210.0, 160.0, 204.0, 187.0, 189.0, 159.0, 168.0, 209.0, 181.0, 159.0, 173.0, 153.0, 175.0, 152.0, 147.0, 174.0, 180.0, 153.0, 176.0, 146.0, 165.0, 154.0, 147.0, 106.0, 147.0, 133.0, 174.0, 148.0, 152.0, 143.0, 173.0, 127.0, 116.0, 130.0, 127.0, 123.0, 143.0, 142.0, 146.0, 123.0, 131.0, 124.0, 138.0, 139.0, 109.0, 107.0, 130.0, 103.0, 121.0, 157.0, 131.0, 148.0, 139.0, 96.0, 120.0, 101.0, 96.0, 102.0, 102.0, 122.0, 105.0, 84.0, 114.0, 117.0, 95.0, 90.0, 106.0, 137.0, 136.0, 131.0, 122.0, 95.0, 111.0, 99.0, 117.0, 119.0, 129.0, 111.0, 104.0, 112.0, 108.0, 102.0, 88.0, 97.0, 120.0, 121.0, 124.0, 96.0, 126.0, 134.0, 122.0, 98.0, 97.0, 115.0, 102.0, 102.0, 128.0, 120.0, 104.0, 104.0, 97.0, 112.0, 104.0, 96.0, 117.0, 97.0, 136.0, 100.0, 92.0, 104.0, 95.0, 111.0, 97.0, 87.0, 108.0, 128.0, 94.0, 111.0, 106.0, 122.0, 99.0, 94.0, 110.0, 104.0, 116.0, 119.0, 114.0, 112.0, 104.0, 104.0, 108.0, 88.0, 105.0, 114.0, 103.0, 105.0, 96.0, 98.0, 92.0, 92.0, 91.0, 102.0, 119.0, 106.0, 86.0, 104.0, 60.0, 110.0, 92.0, 91.0, 80.0, 91.0, 114.0, 106.0, 80.0, 119.0, 117.0, 112.0, 114.0, 98.0, 102.0, 109.0, 101.0, 100.0, 102.0, 126.0, 124.0, 99.0, 112.0, 110.0, 129.0, 111.0, 99.0, 119.0, 101.0, 82.0, 110.0, 84.0, 95.0, 104.0, 96.0, 107.0, 83.0, 114.0, 105.0, 93.0, 104.0, 108.0, 94.0, 99.0, 104.0, 101.0, 88.0, 112.0, 101.0, 101.0, 108.0, 119.0, 118.0, 103.0, 100.0, 107.0, 94.0, 104.0, 118.0, 111.0, 115.0, 100.0, 114.0, 90.0, 110.0, 107.0, 90.0, 91.0, 145.0, 113.0, 112.0, 120.0, 101.0, 98.0, 97.0, 96.0, 109.0, 100.0, 115.0, 120.0, 120.0, 121.0, 128.0, 103.0, 94.0, 104.0, 110.0, 89.0, 102.0, 106.0, 113.0, 117.0, 113.0, 115.0, 93.0, 114.0, 119.0, 132.0, 82.0, 112.0, 105.0, 96.0, 124.0, 107.0, 108.0, 104.0, 145.0, 119.0, 124.0, 115.0, 116.0, 94.0, 130.0, 98.0, 115.0, 117.0, 120.0, 122.0, 122.0, 110.0, 108.0, 87.0, 117.0, 102.0, 123.0, 108.0, 123.0, 107.0, 99.0, 127.0, 94.0, 107.0, 72.0, 102.0, 86.0, 91.0, 94.0, 116.0, 106.0, 120.0, 127.0, 115.0, 124.0, 126.0, 129.0, 117.0, 112.0, 120.0, 119.0, 126.0, 111.0, 119.0, 91.0, 102.0, 95.0, 118.0, 111.0, 99.0, 122.0, 125.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.0215, 180.0215, 180.0215, 180.0215, 180.02148, 180.02147, 180.02145, 180.02135, 180.02115, 180.02094, 180.01958, 180.01897, 180.01846, 180.01624, 180.01547, 180.01477, 180.01425, 180.01407, 180.0143, 180.01436, 180.01428, 180.01392, 180.01378, 180.01396, 180.01488, 180.0161, 180.0174, 180.01826, 180.0192, 180.0206, 180.02252, 180.02473, 180.0269, 180.02902, 180.03149, 180.0345, 180.03801, 180.04181, 180.04555, 180.04977, 180.05466, 180.05992, 180.06546, 180.0715, 180.07837, 180.08574, 180.09346, 180.10184, 180.11102, 180.12125, 180.13229, 180.14392, 180.15607, 180.16887, 180.1824, 180.19664, 180.21123, 180.22636, 180.24251, 180.25967, 180.27742, 180.29587, 180.31598, 180.33707, 180.3582, 180.3808, 180.40411, 180.42862, 180.45422, 180.48024, 180.50642, 180.53325, 180.56082, 180.58878, 180.61742, 180.64685, 180.67635, 180.70671, 180.73753, 180.76909, 180.80096, 180.83255, 180.86522, 180.89883, 180.93253, 180.96713, 181.00252, 181.03773, 181.07297, 181.10829, 181.14496, 181.18279, 181.22028, 181.25752, 181.29439, 181.32959, 181.36458, 181.40088, 181.43741, 181.47369, 181.50917, 181.54332, 181.57774, 181.61334, 181.64902, 181.68596, 181.7242, 181.7617, 181.79843, 181.83513, 181.87192, 181.90961, 181.94727, 181.9857, 182.02441, 182.06326, 182.1035, 182.14424, 182.18398, 182.22302, 182.26132, 182.30066, 182.33942, 182.37904, 182.41917, 182.45876, 182.49632, 182.53271, 182.56963, 182.60735, 182.64554, 182.68359, 182.72183, 182.75928, 182.79482, 182.83173, 182.86961, 182.90521, 182.94044, 182.97412, 183.00899, 183.04352, 183.0809, 183.12045, 183.16031, 183.20035, 183.24016, 183.27913, 183.31721, 183.35562, 183.39336, 183.42928, 183.46495, 183.50055, 183.53683, 183.57225, 183.60655, 183.64061, 183.67566, 183.71036, 183.74536, 183.78122, 183.81776, 183.85562, 183.89389, 183.93182, 183.96855, 184.00623, 184.04614, 184.08539, 184.12434, 184.16336, 184.20358, 184.2431, 184.28152, 184.32024, 184.3553, 184.3905, 184.42917, 184.4704, 184.51273, 184.55392, 184.59485, 184.63615, 184.67656, 184.71397, 184.74928, 184.78352, 184.82126, 184.86098, 184.90076, 184.94235, 184.98337, 185.02277, 185.0623, 185.10294, 185.14499, 185.18594, 185.22719, 185.26956, 185.31255, 185.35408, 185.39359, 185.43069, 185.46863, 185.50841, 185.54842, 185.5876, 185.62738, 185.66747, 185.7076, 185.74796, 185.78799, 185.82808, 185.86952, 185.91144, 185.95245, 185.99278, 186.03255, 186.07283, 186.11411, 186.15575, 186.19742, 186.2375, 186.27637, 186.31621, 186.35637, 186.39667, 186.43544, 186.4731, 186.51167, 186.55107, 186.5916, 186.63014, 186.66568, 186.69972, 186.73563, 186.77632, 186.81931, 186.86119, 186.89891, 186.93753, 186.97639, 187.01602, 187.0556, 187.0981, 187.14053, 187.1834, 187.22716, 187.27185, 187.31763, 187.36372, 187.4113, 187.45898, 187.506, 187.55214, 187.59671, 187.64069, 187.68445, 187.73042, 187.77773, 187.82211, 187.86797, 187.91481, 187.96231, 188.00858, 188.05304, 188.09511, 188.13795, 188.1804, 188.22424, 188.27013, 188.31894, 188.36742, 188.41576, 188.4644, 188.51416, 188.56253, 188.60983, 188.65424, 188.69913, 188.7431, 188.78632, 188.83072, 188.87659, 188.92245, 188.96892, 189.01532, 189.06158, 189.10831, 189.15527, 189.20079, 189.2475, 189.29361, 189.33777, 189.38203, 189.42827, 189.47591, 189.52328, 189.57204, 189.62096, 189.6709, 189.72188, 189.77139, 189.81842, 189.8649, 189.91235, 189.95949, 190.0078, 190.05704, 190.10622, 190.15698, 190.20724, 190.25786, 190.30705, 190.35727, 190.40851, 190.45973, 190.51111, 190.56392, 190.61598, 190.66782, 190.7196, 190.77359, 190.82573, 190.87747, 190.92769, 190.97775, 191.02827, 191.07834, 191.12999, 191.17932, 191.22862, 191.27965, 191.33025, 191.38222, 191.433, 191.48625, 191.53882, 191.59085, 191.64409, 191.698, 191.7515, 191.8065, 191.86282, 191.91794, 191.97198, 192.02602, 192.07971, 192.1337, 192.18675, 192.24236, 192.29745, 192.35396, 192.40863, 192.46198, 192.51579, 192.57161, 192.62778, 192.68323, 192.73868, 192.79523, 192.85144, 192.9077, 192.96512, 193.02281, 193.07899, 193.13582, 193.19206, 193.24911, 193.30396, 193.35805, 193.41168, 193.46552, 193.52077, 193.57597, 193.63229, 193.68961, 193.74706, 193.80554, 193.86365, 193.92087, 193.97789, 194.03809, 194.09793, 194.15579, 194.21254, 194.27122, 194.33063, 194.39035, 194.44989, 194.51079, 194.56964, 194.62762, 194.68622, 194.74329, 194.79973, 194.85442, 194.91043, 194.96838]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.0215, 180.0215, 180.0215, 180.0215, 180.02148, 180.02147, 180.02145, 180.02135, 180.02115, 180.02094, 180.01958, 180.01897, 180.01846, 180.01624, 180.01547, 180.01477, 180.01425, 180.01407, 180.0143, 180.01436, 180.01428, 180.01392, 180.01378, 180.01396, 180.01488, 180.0161, 180.0174, 180.01826, 180.0192, 180.0206, 180.02252, 180.02473, 180.0269, 180.02902, 180.03149, 180.0345, 180.03801, 180.04181, 180.04555, 180.04977, 180.05466, 180.05992, 180.06546, 180.0715, 180.07837, 180.08574, 180.09346, 180.10184, 180.11102, 180.12125, 180.13229, 180.14392, 180.15607, 180.16887, 180.1824, 180.19664, 180.21123, 180.22636, 180.24251, 180.25967, 180.27742, 180.29587, 180.31598, 180.33707, 180.3582, 180.3808, 180.40411, 180.42862, 180.45422, 180.48024, 180.50642, 180.53325, 180.56082, 180.58878, 180.61742, 180.64685, 180.67635, 180.70671, 180.73753, 180.76909, 180.80096, 180.83255, 180.86522, 180.89883, 180.93253, 180.96713, 181.00252, 181.03773, 181.07297, 181.10829, 181.14496, 181.18279, 181.22028, 181.25752, 181.29439, 181.32959, 181.36458, 181.40088, 181.43741, 181.47369, 181.50917, 181.54332, 181.57774, 181.61334, 181.64902, 181.68596, 181.7242, 181.7617, 181.79843, 181.83513, 181.87192, 181.90961, 181.94727, 181.9857, 182.02441, 182.06326, 182.1035, 182.14424, 182.18398, 182.22302, 182.26132, 182.30066, 182.33942, 182.37904, 182.41917, 182.45876, 182.49632, 182.53271, 182.56963, 182.60735, 182.64554, 182.68359, 182.72183, 182.75928, 182.79482, 182.83173, 182.86961, 182.90521, 182.94044, 182.97412, 183.00899, 183.04352, 183.0809, 183.12045, 183.16031, 183.20035, 183.24016, 183.27913, 183.31721, 183.35562, 183.39336, 183.42928, 183.46495, 183.50055, 183.53683, 183.57225, 183.60655, 183.64061, 183.67566, 183.71036, 183.74536, 183.78122, 183.81776, 183.85562, 183.89389, 183.93182, 183.96855, 184.00623, 184.04614, 184.08539, 184.12434, 184.16336, 184.20358, 184.2431, 184.28152, 184.32024, 184.3553, 184.3905, 184.42917, 184.4704, 184.51273, 184.55392, 184.59485, 184.63615, 184.67656, 184.71397, 184.74928, 184.78352, 184.82126, 184.86098, 184.90076, 184.94235, 184.98337, 185.02277, 185.0623, 185.10294, 185.14499, 185.18594, 185.22719, 185.26956, 185.31255, 185.35408, 185.39359, 185.43069, 185.46863, 185.50841, 185.54842, 185.5876, 185.62738, 185.66747, 185.7076, 185.74796, 185.78799, 185.82808, 185.86952, 185.91144, 185.95245, 185.99278, 186.03255, 186.07283, 186.11411, 186.15575, 186.19742, 186.2375, 186.27637, 186.31621, 186.35637, 186.39667, 186.43544, 186.4731, 186.51167, 186.55107, 186.5916, 186.63014, 186.66568, 186.69972, 186.73563, 186.77632, 186.81931, 186.86119, 186.89891, 186.93753, 186.97639, 187.01602, 187.0556, 187.0981, 187.14053, 187.1834, 187.22716, 187.27185, 187.31763, 187.36372, 187.4113, 187.45898, 187.506, 187.55214, 187.59671, 187.64069, 187.68445, 187.73042, 187.77773, 187.82211, 187.86797, 187.91481, 187.96231, 188.00858, 188.05304, 188.09511, 188.13795, 188.1804, 188.22424, 188.27013, 188.31894, 188.36742, 188.41576, 188.4644, 188.51416, 188.56253, 188.60983, 188.65424, 188.69913, 188.7431, 188.78632, 188.83072, 188.87659, 188.92245, 188.96892, 189.01532, 189.06158, 189.10831, 189.15527, 189.20079, 189.2475, 189.29361, 189.33777, 189.38203, 189.42827, 189.47591, 189.52328, 189.57204, 189.62096, 189.6709, 189.72188, 189.77139, 189.81842, 189.8649, 189.91235, 189.95949, 190.0078, 190.05704, 190.10622, 190.15698, 190.20724, 190.25786, 190.30705, 190.35727, 190.40851, 190.45973, 190.51111, 190.56392, 190.61598, 190.66782, 190.7196, 190.77359, 190.82573, 190.87747, 190.92769, 190.97775, 191.02827, 191.07834, 191.12999, 191.17932, 191.22862, 191.27965, 191.33025, 191.38222, 191.433, 191.48625, 191.53882, 191.59085, 191.64409, 191.698, 191.7515, 191.8065, 191.86282, 191.91794, 191.97198, 192.02602, 192.07971, 192.1337, 192.18675, 192.24236, 192.29745, 192.35396, 192.40863, 192.46198, 192.51579, 192.57161, 192.62778, 192.68323, 192.73868, 192.79523, 192.85144, 192.9077, 192.96512, 193.02281, 193.07899, 193.13582, 193.19206, 193.24911, 193.30396, 193.35805, 193.41168, 193.46552, 193.52077, 193.57597, 193.63229, 193.68961, 193.74706, 193.80554, 193.86365, 193.92087, 193.97789, 194.03809, 194.09793, 194.15579, 194.21254, 194.27122, 194.33063, 194.39035, 194.44989, 194.51079, 194.56964, 194.62762, 194.68622, 194.74329, 194.79973, 194.85442, 194.91043, 194.96838]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [25.9357, 1.58651, 1.57374, 1.5753, 1.57369, 1.58365, 1.58825, 1.58527, 1.58564, 1.5777, 1.58419, 1.58585, 1.58154, 1.58741, 1.59392, 1.59071, 1.59711, 1.6014, 1.60351, 1.59396, 1.5899, 1.59645, 1.58704, 1.58712, 1.60341, 1.58462, 1.5838, 1.58964, 1.5977, 1.5914, 1.59087, 1.59805, 1.5927, 1.59042, 1.57661, 1.58906, 1.58372, 1.5783, 1.662, 1.58247, 1.58561, 1.58497, 1.60619, 1.59828, 1.60708, 1.60788, 1.6018, 1.59949, 1.59104, 1.5968, 1.60548, 1.60125, 1.59943, 1.58135, 1.58089, 1.58389, 1.58725, 1.58116, 1.58404, 1.58902, 1.58673, 1.58415, 1.60076, 1.59392, 1.59498, 1.58949, 1.59688, 1.59686, 1.58746, 1.59881, 1.5919, 1.59305, 1.60935, 1.59895, 1.60324, 1.60238, 1.59829, 1.60008, 1.59605, 1.60176, 1.59396, 1.60186, 1.58731, 1.58171, 1.58397, 1.58802, 1.58792, 1.5888, 1.5989, 1.60961, 1.59174, 1.61116, 1.59839, 1.5987, 1.60266, 1.59894, 1.60234, 1.59759, 1.59588, 1.59656, 1.60095, 1.59247, 1.59334, 1.58581, 1.60076, 1.5966, 1.58958, 1.58303, 1.58777, 1.58897, 1.59327, 1.59617, 1.59379, 1.59354, 1.58468, 1.59116, 1.58522, 1.58052, 1.57531, 1.59285, 1.58327, 1.57928, 1.58856, 1.60734, 1.60047, 1.58954, 1.5887, 1.59365, 1.57967, 1.58675, 1.57718, 1.58018, 1.58698, 1.58486, 1.59903, 1.5922, 1.59084, 1.58453, 1.58231, 1.58267, 1.58483, 1.58037, 1.5909, 1.60252, 1.60356, 1.58876, 1.59367, 1.60171, 1.59771, 1.6032, 1.60106, 1.60184, 1.60827, 1.60637, 1.60548, 1.60525, 1.60212, 1.60506, 1.59982, 1.60509, 1.60647, 1.60886, 1.60014, 1.60931, 1.59824, 1.60157, 1.60774, 1.60732, 1.61218, 1.61074, 1.60769, 1.60031, 1.59568, 1.59819, 1.6096, 1.59367, 1.60494, 1.59917, 1.59747, 1.60124, 1.59771, 1.59534, 1.60201, 1.59851, 1.60069, 1.60225, 1.59775, 1.59041, 1.60108, 1.59759, 1.59096, 1.60191, 1.5962, 1.60086, 1.61379, 1.60436, 1.60606, 1.60163, 1.60378, 1.60305, 1.59492, 1.60456, 1.60034, 1.58872, 1.59577, 1.59654, 1.59711, 1.59749, 1.59808, 1.60144, 1.59512, 1.59382, 1.59822, 1.59585, 1.59994, 1.59286, 1.59958, 1.60154, 1.59764, 1.59284, 1.59867, 1.6049, 1.6004, 1.59909, 1.60488, 1.59532, 1.60133, 1.60538, 1.5991, 1.59608, 1.60992, 1.60101, 1.60144, 1.59775, 1.59962, 1.58809, 1.59851, 1.59204, 1.59492, 1.59647, 1.58928, 1.58595, 1.7535, 1.6478, 1.59827, 1.60514, 1.59426, 1.61414, 1.60982, 1.60735, 1.60866, 1.70147, 1.60416, 1.59248, 1.59525, 1.59344, 1.59499, 1.60459, 1.6003, 1.60341, 1.60801, 1.61343, 1.60596, 1.60611, 1.60542, 1.60121, 1.59801, 1.59823, 1.59998, 1.59829, 1.59898, 1.59531, 1.60142, 1.60403, 1.59966, 1.60202, 1.59979, 1.60042, 1.59732, 1.60245, 1.60091, 1.5998, 1.60238, 1.59984, 1.60274, 1.60666, 1.60321, 1.6036, 1.6041, 1.59868, 1.6015, 1.60892, 1.60377, 1.60116, 1.60829, 1.60355, 1.60349, 1.60256, 1.60399, 1.60265, 1.60684, 1.60536, 1.61211, 1.60719, 1.6104, 1.59911, 1.59879, 1.61165, 1.60015, 1.6048, 1.59789, 1.60116, 1.60929, 1.60128, 1.60444, 1.6133, 1.59942, 1.6132, 1.60448, 1.58597, 1.58802, 1.59401, 1.58972, 1.59965, 1.60201, 1.59413, 1.60397, 1.60165, 1.59963, 1.60178, 1.59826, 1.60301, 1.6063, 1.60499, 1.6023, 1.60467, 1.6048, 1.59497, 1.61355, 1.60237, 1.60516, 1.60289, 1.60404, 1.60076, 1.59623, 1.60269, 1.60248, 1.60802, 1.60059, 1.70142, 1.61751, 1.60679, 1.7026, 1.60996, 1.6083, 1.61064, 1.61183, 1.62052, 1.61909, 1.61534, 1.61668, 1.6033, 1.60768, 1.60386, 1.61143, 1.60918, 1.59776, 1.60709, 1.60535, 1.60161, 1.60666, 1.60582, 1.60545, 1.6075, 1.60733, 1.61657, 1.62133, 1.60999, 1.61188, 1.61305, 1.6069, 1.61671, 1.61762, 1.62212, 1.61922, 1.6081, 1.60551, 1.61555, 1.61354, 1.61632, 1.61937, 1.6141, 1.60911, 1.614, 1.61245, 1.61194, 1.6115, 1.60534, 1.60841, 1.60561]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60068]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60068]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [270.6116]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [270.6116]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/model_config.yaml
index 0efe0da30b..61edc36fbe 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_fp8_tp_pp_sp/model_config.yaml
@@ -1,6 +1,7 @@
 ENV_VARS:
   CUDA_DEVICE_MAX_CONNECTIONS: 1
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
 MODEL_ARGS:
@@ -39,13 +40,14 @@ MODEL_ARGS:
   --transformer-impl: transformer_engine
   --tensor-model-parallel-size: 2
   --pipeline-model-parallel-size: 2
+  --sequence-parallel: true
   --deterministic-mode: true
   --no-gradient-accumulation-fusion: true
   --fp8-format: hybrid
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/golden_values.json
new file mode 100644
index 0000000000..6009b31b8c
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [21.99065, 1.57716, 1.57152, 1.59321, 1.5863, 1.57478, 1.58558, 1.58758, 1.58479, 1.57895, 1.58493, 1.59324, 1.58233, 1.56795, 1.58055, 1.56688, 1.5678, 1.56791, 1.63783, 1.57453, 1.56285, 1.57599, 1.58307, 1.58147, 1.58884, 1.58487, 1.58844, 1.57964, 1.57929, 1.58802, 1.58073, 1.58122, 1.58449, 1.66952, 1.599, 1.60139, 1.74695, 1.60068, 1.58989, 1.59432, 1.59007, 1.58621, 1.5794, 1.58271, 1.58306, 1.59677, 1.59288, 1.58275, 1.67584, 1.59164, 1.60367, 1.5998, 1.60414, 1.59758, 1.58882, 1.60425, 1.59006, 1.58295, 1.58588, 1.60115, 1.59394, 1.6001, 1.59693, 1.5944, 1.59722, 1.60347, 1.59248, 1.67877, 1.59416, 1.59784, 1.61277, 1.59908, 1.59639, 1.5935, 1.59862, 1.61381, 1.60093, 1.59916, 1.59139, 1.59544, 1.60373, 1.59931, 1.59729, 1.58924, 1.59278, 1.60393, 1.59751, 1.59588, 1.597, 1.5921, 1.60557, 1.5915, 1.59296, 1.59099, 1.58952, 1.59785, 1.59236, 1.59138, 1.58196, 1.68409, 1.59552, 1.60388, 1.59454, 1.58942, 1.58688, 1.59613, 1.60092, 1.59976, 1.59462, 1.60601, 1.59966, 1.59879, 1.59803, 1.59743, 1.60087, 1.60123, 1.60561, 1.59721, 1.60002, 1.59717, 1.60267, 1.60202, 1.58969, 1.5937, 1.59501, 1.59729, 1.6055, 1.59373, 1.59552, 1.59903, 1.60628, 1.59959, 1.60033, 1.59523, 1.59534, 1.59886, 1.59989, 1.59127, 1.60846, 1.60265, 1.6054, 1.59487, 1.59192, 1.58491, 1.59173, 1.59624, 1.60184, 1.59635, 1.60701, 1.59973, 1.59592, 1.58783, 1.59596, 1.59257, 1.60207, 1.59766, 1.59014, 1.59147, 1.58958, 1.58849, 1.59599, 1.59796, 1.59187, 1.59629, 1.59167, 1.59103, 1.58381, 1.59206, 1.58888, 1.5904, 1.58555, 1.59114, 1.58539, 1.58566, 1.5894, 1.58315, 1.57556, 1.5798, 1.57936, 1.59144, 1.59188, 1.58985, 1.58744, 1.57959, 1.57707, 1.58114, 1.57447, 1.58757, 1.58393, 1.5814, 1.58214, 1.56869, 1.59904, 1.58832, 1.58446, 1.5886, 1.5964, 1.59995, 1.58984, 1.58458, 1.57848, 1.58262, 1.58372, 1.58511, 1.57472, 1.58482, 1.57884, 1.57655, 1.57371, 1.56768, 1.58436, 1.57434, 1.58546, 1.57895, 1.58824, 1.58943, 1.58534, 1.58931, 1.58768, 1.67183, 1.5994, 1.59551, 1.58731, 1.58941, 1.59427, 1.59768, 1.58889, 1.5907, 1.58959, 1.58719, 1.59215, 1.5863, 1.59281, 1.59155, 1.58447, 1.58437, 1.5847, 1.58696, 1.59622, 1.58517, 1.59019, 1.60434, 1.59968, 1.5969, 1.59751, 1.59456, 1.6066, 1.59805, 1.59315, 1.59835, 1.60342, 1.62288, 1.59735, 1.59455, 1.59386, 1.5899, 1.60537, 1.58935, 1.59479, 1.5931, 1.59564, 1.61221, 1.59658, 1.59741, 1.60139, 1.59726, 1.60686, 1.59462, 1.59958, 1.59653, 1.59254, 1.60457, 1.59551, 1.59428, 1.60093, 1.5944, 1.60142, 1.59772, 1.58999, 1.59811, 1.59342, 1.59459, 1.59229, 1.59446, 1.59758, 1.59514, 1.59376, 1.60015, 1.59289, 1.60569, 1.59243, 1.59995, 1.60277, 1.58962, 1.59704, 1.59408, 1.58742, 1.59956, 1.5946, 1.59711, 1.59521, 1.60094, 1.60537, 1.59472, 1.60512, 1.59709, 1.59942, 1.60326, 1.59747, 1.59643, 1.60252, 1.59668, 1.5978, 1.59291, 1.60286, 1.59494, 1.60307, 1.6023, 1.61125, 1.60608, 1.60499, 1.60013, 1.60294, 1.59839, 1.59445, 1.59771, 1.59912, 1.59625, 1.60071, 1.592, 1.59986, 1.59715, 1.59092, 1.5888, 1.58483, 1.58369, 1.58578, 1.58892, 1.58607, 1.57772, 1.58567, 1.58058, 1.57579, 1.58081, 1.57885, 1.57944, 1.5775, 1.57886, 1.58441, 1.64955, 1.57793, 1.57628, 1.57996, 1.60901, 1.5979, 1.59148, 1.58504, 1.58873, 1.61471, 1.61412, 1.59947, 1.59781, 1.59535, 1.61042, 1.60213, 1.59684, 1.59637, 1.59781, 1.60971, 1.59714, 1.58835, 1.59658, 1.5958, 1.5924, 1.59655, 1.59597, 1.60519, 1.60003, 1.61195, 1.61366, 1.6023, 1.60659, 1.59405, 1.60115, 1.6049, 1.6052, 1.60253, 1.59948, 1.5816, 1.59621, 1.58755, 1.59445, 1.59719, 1.59069, 1.60911, 1.59481, 1.59684, 1.60214, 1.59905, 1.60381]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.16126, 0.78048, 0.77638, 0.78285, 0.77945, 0.7768, 0.78398, 0.78215, 0.7833, 0.77542, 0.78468, 0.78711, 0.78251, 0.76662, 0.76894, 0.76826, 0.77171, 0.76847, 0.83221, 0.7706, 0.76442, 0.77548, 0.77966, 0.76518, 0.7854, 0.7799, 0.77136, 0.76634, 0.78834, 0.77019, 0.78986, 0.77045, 0.78652, 0.87018, 0.80011, 0.7944, 0.94182, 0.79666, 0.78564, 0.78708, 0.78355, 0.78735, 0.78535, 0.79227, 0.79173, 0.79116, 0.79578, 0.78576, 0.88058, 0.78541, 0.7905, 0.80177, 0.80159, 0.79536, 0.78436, 0.80424, 0.79113, 0.78133, 0.79513, 0.79725, 0.78505, 0.80445, 0.7974, 0.80505, 0.80566, 0.79011, 0.78303, 0.8828, 0.7992, 0.80046, 0.79496, 0.80104, 0.80208, 0.78598, 0.79918, 0.79817, 0.80692, 0.79948, 0.79832, 0.80065, 0.79953, 0.80613, 0.80349, 0.79995, 0.80406, 0.8022, 0.80453, 0.80228, 0.8056, 0.79734, 0.80242, 0.78707, 0.79319, 0.80876, 0.78925, 0.79762, 0.79177, 0.81095, 0.78559, 0.87702, 0.80826, 0.80874, 0.79998, 0.78873, 0.79623, 0.80044, 0.7965, 0.80088, 0.80451, 0.80617, 0.80803, 0.80736, 0.80357, 0.80072, 0.80574, 0.80861, 0.80081, 0.80256, 0.8016, 0.80416, 0.80062, 0.79705, 0.79613, 0.7934, 0.79423, 0.79439, 0.79639, 0.79437, 0.80375, 0.79641, 0.8075, 0.79693, 0.80388, 0.79802, 0.79685, 0.80158, 0.79875, 0.79886, 0.80926, 0.81104, 0.80752, 0.80381, 0.79608, 0.7893, 0.78982, 0.79582, 0.79985, 0.79486, 0.8058, 0.79802, 0.79424, 0.79685, 0.79506, 0.79473, 0.79858, 0.79203, 0.79193, 0.79375, 0.79263, 0.78662, 0.78983, 0.79242, 0.78834, 0.78866, 0.78847, 0.79475, 0.78474, 0.78928, 0.78727, 0.7942, 0.78678, 0.78404, 0.7855, 0.78669, 0.7807, 0.79077, 0.78107, 0.78201, 0.78183, 0.80216, 0.79952, 0.79773, 0.7904, 0.78485, 0.7784, 0.78943, 0.78644, 0.78928, 0.79161, 0.79481, 0.79068, 0.78383, 0.79727, 0.78767, 0.79378, 0.79855, 0.79573, 0.79906, 0.79796, 0.78811, 0.77833, 0.78832, 0.79352, 0.78682, 0.78545, 0.78929, 0.78422, 0.78978, 0.78901, 0.78354, 0.78883, 0.78807, 0.79656, 0.79382, 0.79009, 0.79261, 0.79204, 0.79399, 0.79138, 0.87044, 0.79415, 0.78856, 0.7904, 0.7891, 0.78842, 0.79047, 0.78866, 0.78816, 0.78669, 0.78557, 0.78863, 0.79242, 0.79337, 0.78575, 0.78866, 0.78509, 0.78346, 0.78462, 0.78704, 0.78025, 0.78234, 0.78547, 0.78832, 0.78406, 0.79176, 0.78752, 0.79148, 0.7926, 0.78905, 0.79623, 0.79876, 0.80189, 0.79329, 0.78938, 0.78571, 0.79206, 0.79022, 0.78916, 0.79198, 0.78965, 0.78841, 0.79706, 0.79681, 0.79422, 0.79582, 0.7978, 0.7929, 0.79692, 0.79951, 0.79613, 0.78441, 0.78081, 0.78582, 0.78913, 0.79294, 0.7902, 0.78677, 0.79445, 0.79001, 0.79247, 0.78884, 0.78757, 0.79082, 0.79372, 0.79339, 0.79117, 0.79464, 0.79238, 0.78456, 0.80253, 0.7832, 0.79582, 0.78585, 0.78817, 0.7996, 0.80334, 0.80038, 0.78266, 0.79835, 0.80583, 0.7884, 0.803, 0.7964, 0.7803, 0.80771, 0.78154, 0.78737, 0.78425, 0.79511, 0.79935, 0.79899, 0.80031, 0.79737, 0.7882, 0.78726, 0.80196, 0.78826, 0.79069, 0.79987, 0.80053, 0.79658, 0.80868, 0.78979, 0.79176, 0.80466, 0.79718, 0.80577, 0.78989, 0.78977, 0.79845, 0.80176, 0.79513, 0.79765, 0.78377, 0.78605, 0.7817, 0.78486, 0.78251, 0.782, 0.77773, 0.78515, 0.78532, 0.7826, 0.78594, 0.7847, 0.78814, 0.78399, 0.78924, 0.78495, 0.85297, 0.78501, 0.78455, 0.78521, 0.79499, 0.78326, 0.78572, 0.78491, 0.78588, 0.79342, 0.79911, 0.79939, 0.79997, 0.78403, 0.79216, 0.80483, 0.79356, 0.79564, 0.79104, 0.79195, 0.79461, 0.79321, 0.78786, 0.79505, 0.78766, 0.78873, 0.7989, 0.79328, 0.79827, 0.79828, 0.79999, 0.80446, 0.80505, 0.79428, 0.80603, 0.80135, 0.79708, 0.78828, 0.78401, 0.78511, 0.79061, 0.7807, 0.78293, 0.7859, 0.78918, 0.79204, 0.7906, 0.79616, 0.79381, 0.7949, 0.79715]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.59311, 0.76076, 0.76217, 0.75984, 0.7615, 0.75659, 0.76053, 0.7532, 0.76274, 0.76117, 0.76101, 0.76233, 0.76144, 0.75668, 0.76922, 0.75609, 0.75913, 0.76116, 0.76025, 0.76541, 0.75884, 0.75825, 0.75703, 0.766, 0.76226, 0.76154, 0.76489, 0.76817, 0.75764, 0.76666, 0.76075, 0.75889, 0.75671, 0.76413, 0.76441, 0.76109, 0.75862, 0.76306, 0.74826, 0.75641, 0.74619, 0.74555, 0.74425, 0.74896, 0.74343, 0.75132, 0.74633, 0.74611, 0.74624, 0.74486, 0.75681, 0.756, 0.75967, 0.7522, 0.74699, 0.75759, 0.75126, 0.74675, 0.75177, 0.75405, 0.7585, 0.75155, 0.75405, 0.75102, 0.75148, 0.75893, 0.74911, 0.74587, 0.75218, 0.74921, 0.76638, 0.74462, 0.7501, 0.7496, 0.74661, 0.7608, 0.75236, 0.74756, 0.74835, 0.74741, 0.75597, 0.74513, 0.75335, 0.74569, 0.74992, 0.75987, 0.73959, 0.74426, 0.7594, 0.74595, 0.75601, 0.74294, 0.74297, 0.75107, 0.74798, 0.75807, 0.74348, 0.75472, 0.74211, 0.7499, 0.7459, 0.75376, 0.74383, 0.74411, 0.74537, 0.74321, 0.75045, 0.74449, 0.75823, 0.74876, 0.74922, 0.75592, 0.75588, 0.75204, 0.74904, 0.74934, 0.76179, 0.74708, 0.74898, 0.7495, 0.749, 0.75109, 0.75134, 0.74604, 0.74742, 0.74319, 0.75078, 0.74752, 0.75245, 0.74673, 0.75517, 0.75235, 0.74881, 0.74945, 0.75053, 0.74903, 0.75641, 0.74336, 0.76521, 0.75829, 0.75724, 0.75492, 0.7561, 0.75292, 0.74603, 0.75381, 0.74787, 0.75257, 0.76831, 0.74923, 0.75133, 0.74595, 0.75539, 0.74856, 0.75247, 0.75168, 0.74839, 0.75531, 0.74901, 0.75107, 0.75151, 0.75163, 0.75496, 0.75207, 0.75274, 0.75371, 0.75218, 0.75324, 0.75429, 0.74775, 0.75082, 0.74975, 0.75003, 0.74514, 0.74798, 0.7422, 0.74955, 0.74687, 0.74432, 0.76318, 0.76862, 0.75695, 0.75138, 0.74947, 0.74824, 0.74949, 0.74673, 0.76097, 0.75456, 0.75612, 0.74619, 0.74667, 0.75557, 0.75602, 0.74867, 0.74532, 0.75908, 0.75984, 0.75566, 0.75544, 0.74912, 0.74344, 0.74466, 0.743, 0.74211, 0.75391, 0.74844, 0.74322, 0.7419, 0.7391, 0.75107, 0.74688, 0.74472, 0.74867, 0.74188, 0.75312, 0.75735, 0.75298, 0.75011, 0.83767, 0.75688, 0.7468, 0.75125, 0.75873, 0.75439, 0.76222, 0.74909, 0.75114, 0.74996, 0.74891, 0.75631, 0.75529, 0.75222, 0.74576, 0.74916, 0.74348, 0.7422, 0.74917, 0.74763, 0.74945, 0.74253, 0.75781, 0.74585, 0.75081, 0.75209, 0.75165, 0.7532, 0.75146, 0.75199, 0.75085, 0.75606, 0.76797, 0.74123, 0.75583, 0.7498, 0.74976, 0.76018, 0.74891, 0.74315, 0.74567, 0.74733, 0.76326, 0.74371, 0.74843, 0.74397, 0.74563, 0.76375, 0.74742, 0.7484, 0.75035, 0.74757, 0.75381, 0.7431, 0.74767, 0.74383, 0.74076, 0.75278, 0.75322, 0.74717, 0.74642, 0.74435, 0.74553, 0.75415, 0.75172, 0.74406, 0.74946, 0.74845, 0.7471, 0.74058, 0.74992, 0.74948, 0.74994, 0.75938, 0.75195, 0.75199, 0.75277, 0.74398, 0.75468, 0.74625, 0.74009, 0.75462, 0.74436, 0.75709, 0.75842, 0.75583, 0.75652, 0.75955, 0.75822, 0.74976, 0.74693, 0.7489, 0.7484, 0.74876, 0.75623, 0.75485, 0.75131, 0.75086, 0.75519, 0.7563, 0.75201, 0.74461, 0.75083, 0.75104, 0.7491, 0.74353, 0.74963, 0.74824, 0.75106, 0.75407, 0.74618, 0.7523, 0.75149, 0.74913, 0.74663, 0.74746, 0.7482, 0.74592, 0.74512, 0.75269, 0.74881, 0.75383, 0.74575, 0.74092, 0.74646, 0.74972, 0.75151, 0.74727, 0.74596, 0.75029, 0.74634, 0.74441, 0.75077, 0.76193, 0.7811, 0.76201, 0.76484, 0.77016, 0.76471, 0.76985, 0.76565, 0.75567, 0.76091, 0.76601, 0.7782, 0.76131, 0.75676, 0.76458, 0.76377, 0.77738, 0.75801, 0.75902, 0.762, 0.75749, 0.75518, 0.75814, 0.7671, 0.76157, 0.76399, 0.77689, 0.76899, 0.76062, 0.76435, 0.76315, 0.75948, 0.77408, 0.75612, 0.76269, 0.75559, 0.76227, 0.77122, 0.76094, 0.76349, 0.7582, 0.75871, 0.77745, 0.76055, 0.76243, 0.76016, 0.76322, 0.76742]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.19292, 0.01741, 0.01488, 0.01641, 0.01712, 0.01701, 0.01724, 0.01612, 0.01735, 0.01689, 0.01449, 0.01795, 0.01495, 0.01541, 0.01502, 0.01516, 0.01428, 0.01451, 0.01769, 0.01847, 0.0169, 0.01788, 0.01813, 0.01751, 0.01774, 0.01679, 0.01619, 0.01655, 0.01654, 0.01696, 0.0174, 0.0185, 0.01671, 0.01581, 0.01697, 0.01627, 0.02111, 0.01585, 0.0176, 0.01783, 0.01799, 0.01548, 0.01578, 0.01602, 0.01539, 0.01659, 0.01748, 0.01708, 0.01454, 0.01909, 0.01622, 0.01722, 0.01943, 0.01822, 0.01639, 0.01887, 0.0157, 0.01802, 0.01601, 0.01682, 0.01679, 0.01666, 0.01696, 0.01447, 0.01725, 0.01735, 0.01643, 0.01884, 0.01609, 0.0185, 0.0184, 0.01703, 0.01561, 0.01899, 0.01693, 0.01673, 0.01557, 0.02037, 0.01648, 0.02182, 0.01581, 0.01883, 0.01486, 0.01422, 0.01602, 0.0206, 0.01692, 0.01644, 0.01443, 0.0164, 0.01772, 0.01699, 0.01792, 0.01841, 0.01616, 0.01914, 0.01786, 0.01399, 0.01385, 0.01298, 0.01984, 0.01393, 0.01641, 0.01237, 0.01672, 0.01523, 0.01481, 0.01312, 0.01514, 0.0141, 0.01688, 0.01659, 0.01531, 0.01306, 0.01415, 0.01307, 0.01504, 0.01566, 0.01521, 0.01304, 0.0151, 0.01337, 0.01578, 0.01428, 0.01733, 0.01324, 0.01568, 0.01651, 0.01314, 0.01407, 0.01374, 0.01429, 0.01421, 0.01802, 0.01439, 0.01347, 0.01541, 0.01301, 0.01489, 0.01769, 0.01406, 0.01394, 0.01544, 0.01425, 0.01399, 0.01414, 0.01541, 0.01538, 0.01478, 0.01476, 0.01498, 0.01626, 0.01614, 0.01516, 0.0146, 0.02163, 0.01496, 0.01399, 0.0156, 0.01517, 0.01657, 0.01525, 0.02091, 0.01583, 0.01574, 0.01726, 0.01555, 0.01523, 0.01459, 0.01318, 0.01563, 0.01531, 0.01592, 0.01602, 0.01375, 0.01616, 0.01854, 0.0199, 0.01523, 0.01384, 0.01396, 0.01413, 0.01587, 0.01384, 0.01554, 0.01277, 0.0125, 0.01321, 0.01511, 0.01439, 0.01651, 0.01382, 0.01689, 0.01614, 0.01571, 0.01361, 0.01704, 0.01534, 0.01385, 0.01423, 0.20705, 0.01218, 0.01233, 0.01727, 0.01275, 0.01244, 0.01327, 0.01272, 0.01371, 0.01665, 0.01392, 0.01222, 0.01222, 0.01188, 0.01265, 0.01482, 0.01632, 0.01649, 0.01702, 0.10117, 0.01844, 0.01611, 0.01574, 0.01967, 0.01779, 0.0181, 0.01873, 0.01598, 0.01615, 0.0136, 0.01405, 0.0131, 0.01348, 0.01358, 0.01592, 0.01254, 0.01772, 0.01503, 0.01408, 0.01322, 0.01435, 0.0158, 0.01713, 0.01512, 0.01582, 0.01578, 0.01584, 0.01532, 0.01652, 0.01516, 0.01295, 0.01398, 0.01359, 0.01339, 0.01358, 0.01304, 0.01422, 0.01314, 0.01282, 0.01422, 0.01411, 0.01529, 0.01575, 0.01454, 0.01377, 0.01423, 0.0158, 0.0128, 0.01659, 0.0174, 0.01592, 0.01617, 0.01462, 0.01415, 0.01495, 0.01263, 0.01928, 0.01701, 0.01799, 0.01302, 0.01537, 0.01683, 0.01358, 0.01378, 0.01553, 0.01478, 0.01516, 0.01864, 0.01487, 0.0145, 0.01315, 0.0163, 0.01453, 0.01978, 0.01808, 0.01337, 0.01516, 0.01483, 0.0141, 0.01325, 0.01391, 0.01431, 0.01452, 0.01452, 0.01284, 0.01318, 0.01339, 0.01336, 0.01442, 0.01234, 0.01424, 0.01284, 0.01762, 0.01661, 0.01281, 0.01962, 0.01329, 0.01356, 0.01369, 0.01291, 0.01345, 0.01577, 0.01307, 0.01371, 0.01245, 0.0144, 0.01266, 0.01493, 0.01942, 0.01384, 0.01403, 0.01338, 0.01325, 0.01563, 0.0138, 0.01307, 0.01453, 0.0157, 0.01517, 0.01449, 0.01345, 0.01482, 0.01389, 0.01533, 0.01504, 0.01529, 0.01484, 0.01361, 0.01578, 0.01436, 0.01584, 0.01282, 0.01395, 0.01777, 0.01465, 0.01446, 0.01422, 0.01426, 0.01624, 0.01786, 0.01661, 0.01321, 0.01562, 0.016, 0.0161, 0.01445, 0.01562, 0.01697, 0.01694, 0.01328, 0.01308, 0.01623, 0.01535, 0.01156, 0.01359, 0.01294, 0.01787, 0.01354, 0.01547, 0.01746, 0.01479, 0.01512, 0.0137, 0.01697, 0.01836, 0.0165, 0.01597, 0.01426, 0.01481, 0.01758, 0.01613, 0.01995, 0.01744, 0.01619, 0.02014, 0.01917, 0.01834, 0.02092, 0.0156, 0.01825]}, "forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [5.93081, 0.02344, 0.02331, 0.02309, 0.02318, 0.02288, 0.02295, 0.02315, 0.02278, 0.02311, 0.02303, 0.02319, 0.02297, 0.02355, 0.0232, 0.02307, 0.02294, 0.02279, 0.02348, 0.02322, 0.02312, 0.02338, 0.02754, 0.02903, 0.02328, 0.02314, 0.02339, 0.02314, 0.02316, 0.02611, 0.02298, 0.02317, 0.02368, 0.02303, 0.02318, 0.0236, 0.02624, 0.02329, 0.02423, 0.02403, 0.02326, 0.02356, 0.02358, 0.02322, 0.02307, 0.02339, 0.02352, 0.02314, 0.02321, 0.02319, 0.02427, 0.02732, 0.02447, 0.02413, 0.02414, 0.02384, 0.02448, 0.02435, 0.0243, 0.02437, 0.02392, 0.02395, 0.02424, 0.0244, 0.02386, 0.02399, 0.02583, 0.02402, 0.02381, 0.02363, 0.02384, 0.02415, 0.02408, 0.02332, 0.02351, 0.02417, 0.02341, 0.02374, 0.0239, 0.02359, 0.02348, 0.02367, 0.02309, 0.02341, 0.02304, 0.02341, 0.02349, 0.02339, 0.02324, 0.02343, 0.02447, 0.02397, 0.02425, 0.02336, 0.02357, 0.02378, 0.02358, 0.02333, 0.02324, 0.02381, 0.02363, 0.02361, 0.02379, 0.023, 0.02331, 0.02406, 0.02303, 0.02381, 0.02338, 0.0233, 0.02375, 0.02361, 0.02338, 0.0254, 0.02366, 0.02346, 0.02319, 0.0231, 0.02322, 0.02336, 0.02359, 0.02301, 0.0232, 0.0231, 0.02325, 0.02535, 0.02543, 0.0249, 0.0258, 0.02421, 0.02631, 0.02569, 0.02546, 0.02523, 0.02374, 0.02369, 0.02287, 0.02328, 0.02335, 0.02342, 0.02348, 0.02584, 0.02846, 0.02333, 0.02325, 0.02317, 0.02344, 0.02362, 0.02449, 0.02398, 0.02331, 0.02313, 0.02338, 0.02374, 0.02377, 0.02343, 0.02294, 0.02316, 0.02278, 0.02313, 0.02341, 0.02344, 0.02325, 0.02347, 0.02341, 0.02425, 0.0234, 0.0236, 0.02348, 0.02328, 0.02322, 0.02797, 0.02349, 0.02368, 0.02483, 0.02541, 0.02365, 0.02349, 0.02286, 0.02337, 0.02361, 0.02351, 0.02501, 0.02329, 0.02303, 0.02332, 0.02369, 0.02402, 0.02326, 0.02743, 0.02371, 0.02333, 0.02452, 0.02852, 0.02423, 0.02431, 0.02363, 0.02347, 0.0234, 0.02355, 0.0171, 0.02364, 0.02374, 0.02365, 0.02307, 0.02279, 0.02328, 0.02362, 0.0233, 0.02395, 0.02325, 0.02349, 0.0286, 0.02347, 0.02365, 0.02351, 0.02314, 0.02283, 0.02321, 0.02365, 0.02339, 0.02363, 0.02445, 0.0234, 0.023, 0.02306, 0.02312, 0.0258, 0.02371, 0.02351, 0.02414, 0.02516, 0.02398, 0.02387, 0.02789, 0.02332, 0.02291, 0.02319, 0.02382, 0.02362, 0.02352, 0.0236, 0.02482, 0.02336, 0.02343, 0.02386, 0.02373, 0.02332, 0.02345, 0.02366, 0.02371, 0.02383, 0.02391, 0.02309, 0.02396, 0.0237, 0.02358, 0.02332, 0.02354, 0.0237, 0.02431, 0.02339, 0.02333, 0.02358, 0.02566, 0.02353, 0.02329, 0.02355, 0.02334, 0.02388, 0.02322, 0.02748, 0.02759, 0.02327, 0.02777, 0.02798, 0.0238, 0.02318, 0.02324, 0.02335, 0.02358, 0.02398, 0.02384, 0.02417, 0.02338, 0.02373, 0.02324, 0.02322, 0.02308, 0.02335, 0.02824, 0.02882, 0.02297, 0.02325, 0.02282, 0.02322, 0.02355, 0.02322, 0.02216, 0.02334, 0.02367, 0.02317, 0.0235, 0.02347, 0.02352, 0.02303, 0.02358, 0.02344, 0.02281, 0.02283, 0.02317, 0.02298, 0.02317, 0.02316, 0.02391, 0.02343, 0.02303, 0.02332, 0.02335, 0.02338, 0.02344, 0.0231, 0.02322, 0.02326, 0.02319, 0.02352, 0.02355, 0.02458, 0.02323, 0.02296, 0.02379, 0.02609, 0.02363, 0.02342, 0.02402, 0.02329, 0.02315, 0.02333, 0.02366, 0.02341, 0.02336, 0.02367, 0.02372, 0.02313, 0.02316, 0.02322, 0.0229, 0.02346, 0.02318, 0.02345, 0.0231, 0.02329, 0.0234, 0.02416, 0.02352, 0.0233, 0.02333, 0.02358, 0.02304, 0.0234, 0.02373, 0.02367, 0.02364, 0.02394, 0.02331, 0.02361, 0.02549, 0.02611, 0.02307, 0.02307, 0.02339, 0.02305, 0.02337, 0.02343, 0.02331, 0.02306, 0.02371, 0.02326, 0.02401, 0.02338, 0.02329, 0.02355, 0.02339, 0.02318, 0.02379, 0.02372, 0.02332, 0.02367, 0.02321, 0.02384, 0.0232, 0.02419, 0.02337, 0.02355, 0.0235, 0.02303, 0.02314, 0.02384, 0.02385, 0.02327]}, "forward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.86591, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00015, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00011, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00014, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00016, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.0001, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00019, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00014, 0.00012, 0.00012, 0.00013, 0.00013, 0.00021, 0.00017, 0.00013, 0.00016, 0.00019, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00015, 0.00017, 0.00012, 0.00012, 0.00013, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00014, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00016, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00012]}, "backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02324, 0.02473, 0.02485, 0.0257, 0.02421, 0.02511, 0.02424, 0.02512, 0.02482, 0.02484, 0.02503, 0.02501, 0.02497, 0.02408, 0.02453, 0.02476, 0.02472, 0.0245, 0.02469, 0.0238, 0.02472, 0.02383, 0.02443, 0.02414, 0.02458, 0.02427, 0.02418, 0.02518, 0.02515, 0.02471, 0.02487, 0.02507, 0.0252, 0.04234, 0.02563, 0.02482, 0.02527, 0.0252, 0.02511, 0.02616, 0.02552, 0.02553, 0.02507, 0.0247, 0.02488, 0.02838, 0.02802, 0.0284, 0.02834, 0.02994, 0.02821, 0.02845, 0.02966, 0.02456, 0.02638, 0.02786, 0.02477, 0.02529, 0.02816, 0.0278, 0.024, 0.02485, 0.02472, 0.02443, 0.02679, 0.02889, 0.02923, 0.02446, 0.02467, 0.02491, 0.02448, 0.02524, 0.0247, 0.02381, 0.02482, 0.02267, 0.02554, 0.02506, 0.02479, 0.02511, 0.02493, 0.02473, 0.02445, 0.02465, 0.02466, 0.02435, 0.02438, 0.02454, 0.02703, 0.02859, 0.02838, 0.02463, 0.02457, 0.02449, 0.02484, 0.02427, 0.02489, 0.02919, 0.02783, 0.02446, 0.02864, 0.02839, 0.02885, 0.02916, 0.02535, 0.02922, 0.02859, 0.02867, 0.02674, 0.02913, 0.02404, 0.02357, 0.02473, 0.02426, 0.0237, 0.02368, 0.02461, 0.02449, 0.02432, 0.02416, 0.02668, 0.0259, 0.02394, 0.02449, 0.0245, 0.02639, 0.02567, 0.02428, 0.02416, 0.0239, 0.0246, 0.0245, 0.02396, 0.02903, 0.02872, 0.02891, 0.0242, 0.0248, 0.02619, 0.02586, 0.02476, 0.02646, 0.02366, 0.02382, 0.02621, 0.02353, 0.02399, 0.02459, 0.02528, 0.02408, 0.0246, 0.02424, 0.028, 0.02928, 0.02952, 0.02881, 0.02431, 0.02457, 0.02417, 0.02444, 0.02498, 0.02401, 0.02303, 0.02437, 0.02609, 0.02618, 0.0244, 0.02636, 0.02449, 0.02888, 0.0291, 0.02963, 0.02433, 0.02789, 0.03263, 0.03258, 0.02856, 0.02595, 0.02508, 0.02561, 0.02568, 0.02893, 0.02364, 0.02454, 0.02431, 0.02431, 0.02435, 0.02361, 0.02447, 0.02415, 0.02557, 0.02442, 0.02388, 0.02473, 0.02836, 0.02932, 0.02902, 0.02464, 0.02588, 0.02525, 0.02855, 0.02485, 0.03232, 0.02798, 0.02376, 0.02448, 0.02369, 0.02397, 0.02417, 0.02554, 0.02412, 0.02385, 0.02386, 0.02939, 0.02461, 0.02396, 0.02522, 0.02468, 0.02408, 0.02344, 0.02381, 0.02444, 0.02442, 0.02457, 0.02446, 0.02491, 0.02474, 0.02468, 0.02463, 0.02469, 0.02618, 0.02458, 0.0243, 0.02465, 0.02436, 0.0246, 0.02381, 0.02431, 0.02492, 0.02438, 0.0239, 0.02778, 0.03263, 0.03015, 0.02489, 0.02497, 0.02827, 0.02851, 0.02831, 0.02923, 0.02893, 0.02474, 0.02501, 0.02434, 0.02523, 0.02437, 0.02557, 0.02446, 0.02462, 0.02479, 0.02496, 0.02454, 0.02469, 0.02509, 0.02486, 0.02485, 0.02426, 0.02434, 0.025, 0.02506, 0.02464, 0.02457, 0.02548, 0.0244, 0.025, 0.02478, 0.0246, 0.025, 0.02481, 0.02465, 0.02469, 0.02502, 0.02443, 0.02451, 0.025, 0.02468, 0.02437, 0.02501, 0.02475, 0.02536, 0.02455, 0.02462, 0.02512, 0.02448, 0.0247, 0.02447, 0.02432, 0.02473, 0.02472, 0.02439, 0.02441, 0.02485, 0.02461, 0.02454, 0.02434, 0.02462, 0.02469, 0.02464, 0.02438, 0.02452, 0.02463, 0.02444, 0.02442, 0.02471, 0.02629, 0.02488, 0.02491, 0.02465, 0.02437, 0.02469, 0.02484, 0.02511, 0.02481, 0.02578, 0.02498, 0.02521, 0.02506, 0.02571, 0.02539, 0.02521, 0.02412, 0.0257, 0.02473, 0.02452, 0.02527, 0.0256, 0.02517, 0.02489, 0.0251, 0.02453, 0.02495, 0.02483, 0.02495, 0.02445, 0.02472, 0.02508, 0.02487, 0.02471, 0.02495, 0.02544, 0.02447, 0.025, 0.02531, 0.02509, 0.02923, 0.02837, 0.02804, 0.02863, 0.03514, 0.02454, 0.02525, 0.02518, 0.02502, 0.02481, 0.02521, 0.02523, 0.02482, 0.02487, 0.02487, 0.02585, 0.02467, 0.02474, 0.02498, 0.02461, 0.02536, 0.02543, 0.02452, 0.02512, 0.02501, 0.02421, 0.02508, 0.02507, 0.02588, 0.02699, 0.02457, 0.02568, 0.0256, 0.02542, 0.02475, 0.02461, 0.02444, 0.0296, 0.02899, 0.02863, 0.02732, 0.02767, 0.02899, 0.02482, 0.02467, 0.02404]}, "backward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00015, 0.00019, 0.00016, 0.00019, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00023, 0.00016, 0.00017, 0.00017, 0.00018, 0.00019, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00018, 0.00016, 0.00018, 0.00016, 0.00016, 0.00018, 0.00016, 0.00017, 0.00016, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00017, 0.00018, 0.0002, 0.00016, 0.00019, 0.00017, 0.00021, 0.00016, 0.00018, 0.00019, 0.00016, 0.00017, 0.00017, 0.00018, 0.0002, 0.00016, 0.00016, 0.00016, 0.00016, 0.00019, 0.00017, 0.00017, 0.00018, 0.00019, 0.00017, 0.00019, 0.00016, 0.00017, 0.00018, 0.00017, 0.00019, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00017, 0.00016, 0.00018, 0.00016, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00017, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00024, 0.00017, 0.00018, 0.00016, 0.00016, 0.00019, 0.00019, 0.00018, 0.00026, 0.00017, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00017, 0.00018, 0.00017, 0.00016, 0.00016, 0.00018, 0.00019, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00021, 0.00016, 0.00018, 0.00019, 0.00022, 0.00017, 0.00016, 0.00017, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00016, 0.00019, 0.00018, 0.00018, 0.00017, 0.00019, 0.00017, 0.00019, 0.00018, 0.00018, 0.00016, 0.00017, 0.00016, 0.00016, 0.00018, 0.00017, 0.00016, 0.00029, 0.00017, 0.00019, 0.0002, 0.00016, 0.00019, 0.00032, 0.00019, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.00023, 0.00018, 0.00018, 0.00018, 0.00017, 0.00019, 0.00018, 0.00016, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.0002, 0.00016, 0.0002, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00016, 0.00018, 0.00016, 0.00017, 0.00019, 0.00018, 0.00016, 0.00019, 0.00022, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00016, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00019, 0.00016, 0.00018, 0.00016, 0.00017, 0.00017, 0.00026, 0.00016, 0.00016, 0.00019, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00017, 0.00017, 0.00016, 0.00019, 0.00018, 0.00017, 0.00016, 0.00018, 0.00016, 0.00016, 0.00016, 0.00018, 0.00016, 0.00019, 0.00019, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00016, 0.00018, 0.00018, 0.00022, 0.00016, 0.00018, 0.00018, 0.00017, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00016, 0.00019, 0.00016, 0.00018, 0.00017, 0.00017, 0.00016, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00016, 0.00016, 0.00017, 0.00016, 0.00018, 0.00016, 0.00017, 0.00019, 0.00017, 0.00018, 0.00019, 0.00019, 0.00018, 0.00016, 0.00016, 0.00017, 0.00018, 0.00016, 0.00019, 0.00016, 0.00016, 0.00016, 0.00016, 0.00016, 0.00019, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00016, 0.00016, 0.0002, 0.00018, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.0003, 0.00016, 0.00018, 0.00018, 0.00016, 0.00019, 0.00018, 0.00019, 0.00016, 0.00016, 0.00016, 0.00018, 0.00019, 0.00018, 0.00018, 0.00017, 0.00018, 0.00019, 0.00017, 0.00016, 0.00018, 0.00017, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00016, 0.00018, 0.00019, 0.00017, 0.00018, 0.00018, 0.00017, 0.00016, 0.00035, 0.00022, 0.00019, 0.00018, 0.00018, 0.00017, 0.00016, 0.00017]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [7.52895, 0.10767, 0.10288, 0.12221, 0.10839, 0.10916, 0.11683, 0.11949, 0.11244, 0.10662, 0.11634, 0.12145, 0.11448, 0.10239, 0.10115, 0.10144, 0.10622, 0.1006, 0.1586, 0.10078, 0.09436, 0.10994, 0.11246, 0.10473, 0.11165, 0.11062, 0.10864, 0.10698, 0.11094, 0.1123, 0.11651, 0.11274, 0.11336, 0.17984, 0.1238, 0.12939, 0.27709, 0.1391, 0.13093, 0.12511, 0.13066, 0.1225, 0.11928, 0.11852, 0.12105, 0.1235, 0.12183, 0.11095, 0.20461, 0.11574, 0.12325, 0.12774, 0.1342, 0.12396, 0.11854, 0.1264, 0.11539, 0.11273, 0.1179, 0.13162, 0.11525, 0.13348, 0.13, 0.12472, 0.13424, 0.1156, 0.11969, 0.21123, 0.12519, 0.12897, 0.136, 0.13444, 0.12965, 0.12283, 0.13807, 0.13035, 0.12784, 0.13095, 0.12328, 0.12278, 0.1242, 0.13846, 0.1251, 0.11622, 0.12258, 0.12174, 0.12831, 0.12841, 0.12632, 0.11745, 0.12732, 0.12029, 0.13155, 0.12567, 0.11834, 0.12549, 0.12416, 0.12349, 0.11452, 0.20614, 0.12415, 0.11944, 0.12148, 0.11366, 0.12373, 0.12834, 0.11722, 0.11892, 0.11557, 0.12715, 0.12886, 0.12057, 0.12682, 0.12601, 0.13364, 0.12815, 0.12626, 0.1317, 0.12917, 0.12301, 0.12818, 0.12239, 0.12231, 0.12391, 0.12264, 0.1209, 0.12986, 0.12429, 0.11971, 0.12228, 0.12907, 0.12399, 0.12889, 0.11751, 0.11734, 0.11985, 0.12419, 0.11939, 0.12896, 0.13183, 0.13356, 0.12001, 0.12131, 0.11604, 0.11794, 0.12429, 0.1355, 0.12631, 0.13817, 0.12757, 0.12565, 0.12479, 0.12459, 0.11863, 0.12603, 0.11965, 0.11957, 0.11941, 0.12277, 0.12152, 0.13238, 0.12899, 0.12039, 0.12936, 0.12185, 0.12027, 0.11834, 0.12565, 0.12003, 0.12064, 0.11734, 0.11796, 0.11982, 0.11829, 0.11018, 0.11427, 0.10291, 0.11078, 0.11775, 0.12251, 0.11736, 0.12288, 0.11757, 0.10965, 0.1101, 0.1111, 0.10524, 0.11035, 0.1194, 0.10687, 0.1104, 0.1029, 0.11414, 0.11835, 0.11073, 0.10671, 0.11471, 0.11713, 0.11142, 0.11427, 0.10551, 0.11576, 0.10811, 0.12352, 0.11089, 0.10827, 0.11418, 0.11243, 0.11291, 0.10774, 0.10575, 0.10895, 0.11133, 0.10168, 0.11589, 0.11188, 0.11403, 0.12083, 0.12527, 0.20209, 0.12301, 0.12835, 0.1167, 0.12035, 0.12158, 0.11749, 0.11785, 0.11663, 0.11859, 0.11189, 0.11229, 0.11518, 0.1205, 0.11283, 0.11679, 0.11705, 0.11627, 0.12181, 0.12372, 0.12191, 0.12006, 0.1168, 0.12252, 0.11718, 0.12814, 0.12688, 0.12696, 0.12607, 0.12079, 0.13508, 0.13166, 0.13101, 0.12769, 0.12321, 0.12875, 0.12726, 0.12271, 0.12496, 0.13106, 0.12712, 0.12831, 0.11758, 0.13314, 0.13148, 0.13269, 0.13383, 0.1235, 0.1316, 0.14168, 0.13684, 0.12388, 0.11908, 0.12703, 0.12329, 0.12975, 0.12484, 0.11743, 0.13142, 0.12276, 0.12584, 0.12278, 0.12351, 0.12006, 0.1275, 0.12997, 0.12275, 0.12374, 0.1258, 0.12674, 0.1382, 0.11985, 0.12902, 0.11699, 0.12694, 0.12671, 0.12528, 0.12577, 0.12335, 0.12793, 0.12913, 0.12309, 0.13132, 0.12457, 0.12253, 0.11803, 0.11645, 0.12181, 0.12507, 0.12528, 0.12214, 0.12812, 0.12471, 0.11918, 0.12456, 0.12769, 0.12304, 0.12153, 0.11907, 0.13148, 0.13103, 0.13068, 0.13318, 0.12552, 0.12933, 0.13261, 0.12839, 0.13023, 0.12205, 0.12863, 0.12765, 0.12548, 0.12592, 0.12495, 0.12574, 0.12193, 0.12065, 0.12433, 0.12257, 0.11243, 0.11188, 0.11552, 0.11773, 0.11637, 0.1131, 0.11535, 0.11323, 0.11728, 0.11383, 0.11656, 0.18458, 0.11533, 0.1158, 0.11306, 0.12884, 0.12649, 0.12032, 0.11208, 0.11803, 0.13436, 0.14069, 0.12596, 0.12808, 0.12036, 0.127, 0.12774, 0.12746, 0.13166, 0.1288, 0.11946, 0.12914, 0.12045, 0.1215, 0.117, 0.11498, 0.11583, 0.11774, 0.12264, 0.12134, 0.12257, 0.12649, 0.1233, 0.12733, 0.11514, 0.12185, 0.12051, 0.13736, 0.13171, 0.13031, 0.11491, 0.11951, 0.10565, 0.11503, 0.1165, 0.11394, 0.11312, 0.11865, 0.11953, 0.12351, 0.12231, 0.12042]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3.33774, 0.00722, 0.00727, 0.01025, 0.00728, 0.00714, 0.00814, 0.00897, 0.00966, 0.00746, 0.00801, 0.00911, 0.00716, 0.01132, 0.00906, 0.00969, 0.00832, 0.01171, 0.00765, 0.00889, 0.00886, 0.01056, 0.00822, 0.01186, 0.00789, 0.00921, 0.01483, 0.01149, 0.00732, 0.00899, 0.00802, 0.00967, 0.01211, 0.00836, 0.00778, 0.0097, 0.00744, 0.00738, 0.00799, 0.00783, 0.00895, 0.00733, 0.00808, 0.00821, 0.00953, 0.00947, 0.00803, 0.00716, 0.0083, 0.01092, 0.01169, 0.01197, 0.01099, 0.0139, 0.01319, 0.01223, 0.00743, 0.01124, 0.01269, 0.01365, 0.01106, 0.01186, 0.01247, 0.01377, 0.01372, 0.00895, 0.00817, 0.0122, 0.00886, 0.01409, 0.01218, 0.0116, 0.01184, 0.01054, 0.0083, 0.01112, 0.01398, 0.01443, 0.01304, 0.01159, 0.01508, 0.01227, 0.01243, 0.00996, 0.01336, 0.0103, 0.0121, 0.00939, 0.01351, 0.0109, 0.0119, 0.00743, 0.01152, 0.01082, 0.0077, 0.013, 0.00863, 0.01128, 0.00747, 0.10318, 0.00737, 0.01277, 0.0074, 0.00766, 0.00929, 0.00731, 0.00777, 0.00773, 0.01305, 0.01203, 0.01277, 0.01218, 0.01038, 0.01189, 0.01149, 0.01182, 0.01209, 0.0087, 0.01115, 0.0143, 0.01389, 0.01471, 0.01226, 0.01046, 0.01269, 0.01445, 0.0131, 0.01159, 0.01285, 0.01374, 0.01248, 0.01373, 0.01412, 0.01487, 0.01463, 0.0142, 0.01491, 0.01425, 0.01332, 0.01294, 0.01394, 0.01396, 0.01223, 0.01179, 0.01522, 0.01396, 0.01383, 0.01262, 0.0137, 0.01453, 0.01605, 0.01203, 0.01365, 0.01102, 0.01296, 0.01149, 0.01352, 0.0141, 0.01337, 0.01015, 0.01142, 0.01244, 0.01056, 0.01302, 0.0136, 0.01251, 0.014, 0.01398, 0.01294, 0.01334, 0.01177, 0.01235, 0.01091, 0.01036, 0.01476, 0.01084, 0.01117, 0.01139, 0.01169, 0.01222, 0.01155, 0.0115, 0.01538, 0.01662, 0.01196, 0.01265, 0.01353, 0.0155, 0.01451, 0.01302, 0.01135, 0.01115, 0.01301, 0.01401, 0.01239, 0.01337, 0.0134, 0.01449, 0.01454, 0.01499, 0.02199, 0.01511, 0.01449, 0.01437, 0.01499, 0.01473, 0.01696, 0.01373, 0.01165, 0.01224, 0.01255, 0.01026, 0.01816, 0.01732, 0.01392, 0.01205, 0.01326, 0.012, 0.0125, 0.09407, 0.01373, 0.01234, 0.01352, 0.01298, 0.01393, 0.01293, 0.01272, 0.01269, 0.00988, 0.01398, 0.01371, 0.01512, 0.00926, 0.01203, 0.00886, 0.01072, 0.01094, 0.01129, 0.01236, 0.01167, 0.01127, 0.0134, 0.01164, 0.01227, 0.01086, 0.01128, 0.01424, 0.01338, 0.01286, 0.01139, 0.0124, 0.01253, 0.01306, 0.0104, 0.01044, 0.00925, 0.01349, 0.0106, 0.01304, 0.013, 0.01652, 0.01247, 0.01259, 0.01119, 0.01241, 0.01609, 0.01301, 0.01673, 0.01245, 0.01358, 0.01293, 0.01395, 0.01222, 0.01281, 0.01194, 0.01332, 0.01097, 0.01369, 0.01398, 0.0117, 0.01357, 0.0128, 0.01277, 0.01159, 0.01226, 0.01271, 0.0131, 0.01357, 0.0123, 0.01025, 0.01114, 0.01335, 0.01274, 0.00948, 0.01342, 0.01348, 0.01171, 0.01274, 0.01313, 0.01262, 0.01167, 0.00993, 0.01158, 0.0107, 0.01309, 0.01347, 0.015, 0.01426, 0.01127, 0.01224, 0.0128, 0.01251, 0.01492, 0.01369, 0.01553, 0.01256, 0.01398, 0.01419, 0.01663, 0.01442, 0.01314, 0.01126, 0.01132, 0.01161, 0.01215, 0.01208, 0.01721, 0.01103, 0.01311, 0.00802, 0.01029, 0.01351, 0.00888, 0.01039, 0.00882, 0.00933, 0.00881, 0.00926, 0.01082, 0.01021, 0.00961, 0.01001, 0.00836, 0.00918, 0.01044, 0.01016, 0.00966, 0.00991, 0.01218, 0.07892, 0.00899, 0.01009, 0.01201, 0.00867, 0.01068, 0.01049, 0.01158, 0.01334, 0.0109, 0.01304, 0.00961, 0.01538, 0.01469, 0.01646, 0.00905, 0.01059, 0.01386, 0.01332, 0.01461, 0.01223, 0.01253, 0.0166, 0.01015, 0.01471, 0.01602, 0.01097, 0.01225, 0.01068, 0.01085, 0.01135, 0.00802, 0.00878, 0.01148, 0.01009, 0.00941, 0.00919, 0.01177, 0.00968, 0.01046, 0.00955, 0.01107, 0.00923, 0.00916, 0.00864, 0.01069, 0.01075, 0.00939, 0.01202, 0.00876, 0.01073]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0012, 0.00075, 0.00074, 0.00352, 0.00166, 0.00076, 0.00077, 0.00076, 0.00319, 0.00077, 0.00076, 0.00445, 0.00077, 0.00075, 0.00153, 0.00077, 0.00076, 0.00076, 0.00076, 0.00077, 0.00076, 0.00075, 0.00076, 0.00075, 0.00077, 0.00075, 0.00077, 0.00075, 0.00077, 0.00077, 0.00075, 0.00076, 0.00076, 0.00076, 0.00076, 0.00076, 0.00077, 0.00076, 0.00076, 0.00077, 0.00078, 0.00076, 0.00077, 0.00076, 0.00076, 0.00429, 0.00076, 0.00076, 0.00076, 0.00078, 0.00077, 0.00077, 0.00078, 0.00077, 0.0008, 0.00079, 0.00079, 0.00077, 0.00078, 0.00078, 0.00079, 0.00519, 0.00079, 0.00078, 0.00077, 0.00078, 0.00079, 0.00079, 0.00079, 0.00077, 0.00079, 0.00079, 0.00079, 0.00078, 0.00078, 0.00078, 0.00077, 0.00079, 0.00079, 0.00079, 0.00078, 0.00078, 0.00077, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00083, 0.00306, 0.00078, 0.00076, 0.00078, 0.00078, 0.00078, 0.00077, 0.00078, 0.0008, 0.00079, 0.00079, 0.00077, 0.00079, 0.00078, 0.00078, 0.00081, 0.00335, 0.00078, 0.00079, 0.0008, 0.00078, 0.00079, 0.00079, 0.00078, 0.00077, 0.00079, 0.00078, 0.00079, 0.0008, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00077, 0.00078, 0.00078, 0.00077, 0.00077, 0.00078, 0.00077, 0.00078, 0.00077, 0.00078, 0.00077, 0.00077, 0.00077, 0.00079, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00077, 0.00078, 0.00078, 0.00079, 0.00086, 0.00079, 0.00078, 0.00079, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.0008, 0.0008, 0.00079, 0.00078, 0.00079, 0.00078, 0.00078, 0.00082, 0.00081, 0.00083, 0.00078, 0.00077, 0.00079, 0.00082, 0.0008, 0.00077, 0.00076, 0.00077, 0.00078, 0.00077, 0.00078, 0.00077, 0.00077, 0.00077, 0.00078, 0.00077, 0.00077, 0.00077, 0.00077, 0.00078, 0.00078, 0.00077, 0.00077, 0.00078, 0.00082, 0.00083, 0.00078, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00077, 0.00078, 0.00079, 0.00078, 0.00452, 0.00077, 0.00078, 0.00077, 0.00077, 0.0008, 0.00078, 0.00079, 0.00079, 0.00078, 0.00223, 0.00078, 0.00077, 0.00077, 0.00079, 0.00078, 0.00078, 0.00078, 0.00295, 0.00077, 0.00077, 0.00077, 0.00077, 0.00077, 0.00076, 0.00077, 0.0042, 0.00081, 0.00079, 0.00087, 0.00078, 0.00078, 0.00078, 0.00078, 0.00076, 0.00078, 0.0008, 0.00076, 0.00079, 0.00077, 0.00078, 0.00077, 0.00077, 0.00077, 0.00078, 0.00078, 0.00077, 0.00078, 0.00077, 0.00077, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00076, 0.00076, 0.00077, 0.00077, 0.00077, 0.00077, 0.00078, 0.00079, 0.00085, 0.00078, 0.00078, 0.00077, 0.00079, 0.00079, 0.00079, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00077, 0.00078, 0.00077, 0.00077, 0.00077, 0.00079, 0.00079, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00079, 0.00077, 0.00078, 0.00078, 0.00077, 0.00077, 0.00078, 0.00077, 0.00077, 0.00079, 0.00079, 0.00077, 0.00077, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00079, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00077, 0.00079, 0.00078, 0.00077, 0.00079, 0.00078, 0.00078, 0.00077, 0.00077, 0.0008, 0.00078, 0.00078, 0.00079, 0.00077, 0.00079, 0.00077, 0.00077, 0.00077, 0.00079, 0.00078, 0.00078, 0.00078, 0.00083, 0.0009, 0.00079, 0.00082, 0.0008, 0.0008, 0.00078, 0.00077, 0.00077, 0.00078, 0.00078, 0.00079, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.0008, 0.00079, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00078, 0.00077, 0.00084, 0.00077, 0.00077, 0.00077, 0.0008, 0.00078, 0.00078, 0.00077, 0.00078, 0.00153, 0.00078, 0.00078, 0.00076]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00036, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00033, 0.00032, 0.00032, 0.00031, 0.00032, 0.00034, 0.00032, 0.00031, 0.00037, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00034, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00034, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00034, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00031, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00034, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00034, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00031, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00033, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00032, 0.00032, 0.00031, 0.00032, 0.00031, 0.00031, 0.00031, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00032, 0.00031]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.22391, 0.00071, 0.00073, 0.0009, 0.00073, 0.00075, 0.00074, 0.00093, 0.00097, 0.00072, 0.00071, 0.00084, 0.00088, 0.00075, 0.00086, 0.00072, 0.00072, 0.00071, 0.00072, 0.00073, 0.00072, 0.00072, 0.00073, 0.00073, 0.00072, 0.00072, 0.00072, 0.00072, 0.00071, 0.0007, 0.00072, 0.00071, 0.00072, 0.00072, 0.00071, 0.00071, 0.00074, 0.00072, 0.00074, 0.00073, 0.00073, 0.00075, 0.00074, 0.00072, 0.00072, 0.00073, 0.0009, 0.00081, 0.00071, 0.00073, 0.00073, 0.00071, 0.00074, 0.00084, 0.00072, 0.00072, 0.00083, 0.00072, 0.00073, 0.00072, 0.0009, 0.00072, 0.00072, 0.00072, 0.00074, 0.00072, 0.00073, 0.00073, 0.00073, 0.00072, 0.00074, 0.00075, 0.00072, 0.00073, 0.00073, 0.00072, 0.00073, 0.00074, 0.00073, 0.00072, 0.00073, 0.00074, 0.00073, 0.00074, 0.00073, 0.00073, 0.00073, 0.00072, 0.00072, 0.00071, 0.00074, 0.00093, 0.00074, 0.00072, 0.00072, 0.00072, 0.00072, 0.00069, 0.00084, 0.00071, 0.00073, 0.00073, 0.0008, 0.00086, 0.00098, 0.00092, 0.00099, 0.00087, 0.00096, 0.00093, 0.00073, 0.00074, 0.00072, 0.00072, 0.00072, 0.00074, 0.00072, 0.00072, 0.00072, 0.00073, 0.00073, 0.00073, 0.00072, 0.00073, 0.00072, 0.00073, 0.00073, 0.00072, 0.00073, 0.00077, 0.00075, 0.00074, 0.00087, 0.00072, 0.00073, 0.00072, 0.00073, 0.00082, 0.00081, 0.00074, 0.00074, 0.00073, 0.00072, 0.00072, 0.00074, 0.00073, 0.00071, 0.00075, 0.00076, 0.00072, 0.00085, 0.00072, 0.00073, 0.00072, 0.00074, 0.00082, 0.00097, 0.00073, 0.00072, 0.00072, 0.00073, 0.00073, 0.00073, 0.00072, 0.00072, 0.00073, 0.00073, 0.00073, 0.00077, 0.00072, 0.00073, 0.00086, 0.00087, 0.00073, 0.00093, 0.00084, 0.00097, 0.00089, 0.00074, 0.00074, 0.00087, 0.00093, 0.00087, 0.00073, 0.00072, 0.00074, 0.00072, 0.00074, 0.00074, 0.00074, 0.00073, 0.00072, 0.00093, 0.00074, 0.00073, 0.00075, 0.00085, 0.00073, 0.00072, 0.00072, 0.00073, 0.00092, 0.00074, 0.00088, 0.00073, 0.00074, 0.00073, 0.00073, 0.00072, 0.00072, 0.00075, 0.00073, 0.00072, 0.00081, 0.00073, 0.00073, 0.00071, 0.00072, 0.00071, 0.00071, 0.00072, 0.00074, 0.00072, 0.00073, 0.00093, 0.00072, 0.00074, 0.00072, 0.00073, 0.00071, 0.00074, 0.00074, 0.00087, 0.00086, 0.00072, 0.00072, 0.00074, 0.00072, 0.00074, 0.00072, 0.00079, 0.00095, 0.00083, 0.00071, 0.00093, 0.00088, 0.00072, 0.00072, 0.00073, 0.00071, 0.00075, 0.00091, 0.00072, 0.00071, 0.00072, 0.00073, 0.0007, 0.00072, 0.00074, 0.00072, 0.00074, 0.00073, 0.00075, 0.00073, 0.00073, 0.00072, 0.00073, 0.00073, 0.00071, 0.00074, 0.00072, 0.00071, 0.00071, 0.00073, 0.00072, 0.00073, 0.00073, 0.00071, 0.00074, 0.00072, 0.00073, 0.00073, 0.0007, 0.00072, 0.00072, 0.00072, 0.00073, 0.00074, 0.00072, 0.00074, 0.00073, 0.00073, 0.00074, 0.0007, 0.00072, 0.00072, 0.00073, 0.00074, 0.00071, 0.00073, 0.00072, 0.00071, 0.00073, 0.00071, 0.00073, 0.00072, 0.00074, 0.00071, 0.00073, 0.00071, 0.00073, 0.00073, 0.00071, 0.0007, 0.00072, 0.00072, 0.00073, 0.00072, 0.00071, 0.00072, 0.00073, 0.00074, 0.00071, 0.00074, 0.00071, 0.00073, 0.00072, 0.00073, 0.00073, 0.00071, 0.00073, 0.00072, 0.00073, 0.00074, 0.00074, 0.00071, 0.00072, 0.00072, 0.00074, 0.00072, 0.00073, 0.00072, 0.00074, 0.00072, 0.00073, 0.00073, 0.00073, 0.00073, 0.00074, 0.00074, 0.00075, 0.00072, 0.00073, 0.00097, 0.00103, 0.00091, 0.00097, 0.00092, 0.00088, 0.00072, 0.00071, 0.00073, 0.00074, 0.00073, 0.00075, 0.0007, 0.00072, 0.00072, 0.00072, 0.00071, 0.00073, 0.00072, 0.00074, 0.00072, 0.00073, 0.00074, 0.00073, 0.00074, 0.00073, 0.00072, 0.00073, 0.00074, 0.00074, 0.00072, 0.00075, 0.0007, 0.00072, 0.00076, 0.00073, 0.00072, 0.00072, 0.00094, 0.00082, 0.00087, 0.00071, 0.00071, 0.00096, 0.00083, 0.00089, 0.00089]}, "params-all-gather-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00024, 0.00025, 0.00024, 0.00043, 0.00027, 0.00024, 0.00024, 0.00024, 0.00035, 0.00024, 0.00024, 0.0004, 0.00025, 0.00024, 0.0003, 0.00025, 0.00024, 0.00024, 0.00024, 0.00025, 0.00024, 0.00025, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00024, 0.00025, 0.00025, 0.00026, 0.00024, 0.00024, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00024, 0.00024, 0.00024, 0.0003, 0.00025, 0.00025, 0.00025, 0.00025, 0.00042, 0.00025, 0.00027, 0.00025, 0.00048, 0.00025, 0.00026, 0.00025, 0.00025, 0.00026, 0.00026, 0.00056, 0.00026, 0.00043, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00033, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00028, 0.00043, 0.00026, 0.00034, 0.0003, 0.00025, 0.0003, 0.00024, 0.00025, 0.00026, 0.00026, 0.00024, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00026, 0.00024, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00024, 0.00025, 0.00026, 0.00024, 0.00024, 0.00025, 0.00028, 0.00025, 0.00025, 0.00025, 0.00025, 0.00028, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00027, 0.00025, 0.00025, 0.00026, 0.00026, 0.00027, 0.00025, 0.00026, 0.00025, 0.00026, 0.00046, 0.00025, 0.00025, 0.00025, 0.00025, 0.00045, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00024, 0.00027, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00043, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00032, 0.0005, 0.00025, 0.00024, 0.0005, 0.00038, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00042, 0.00025, 0.0004, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00027, 0.00025, 0.00026, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00026, 0.00025, 0.00026, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00039, 0.00029, 0.00026, 0.00025, 0.00025, 0.00033, 0.00025, 0.00025, 0.00026, 0.00026, 0.00027, 0.00033, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00025, 0.00025, 0.00024, 0.00024, 0.00024, 0.00026, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00024, 0.00024, 0.00024, 0.00025, 0.00025, 0.00044, 0.00044, 0.00046, 0.00041, 0.00047, 0.00026, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00026, 0.00024, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00025, 0.00024, 0.00025, 0.00025, 0.00026, 0.00025, 0.00026, 0.00025, 0.00025, 0.00026, 0.00025, 0.00025, 0.00024, 0.00043, 0.00026, 0.00053, 0.00025, 0.00026, 0.00025, 0.00028, 0.00042, 0.00025, 0.00025]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00041, 0.00039, 0.00039, 0.00041, 0.00042, 0.0004, 0.00041, 0.0004, 0.0004, 0.0004, 0.0004, 0.00054, 0.0004, 0.0004, 0.00056, 0.00042, 0.0004, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00042, 0.0004, 0.0004, 0.00041, 0.00041, 0.00041, 0.0004, 0.00041, 0.0004, 0.00041, 0.00041, 0.00041, 0.0004, 0.00041, 0.00042, 0.00041, 0.00042, 0.00041, 0.00042, 0.00042, 0.0004, 0.00041, 0.00042, 0.00042, 0.0004, 0.00041, 0.00043, 0.00041, 0.00042, 0.00041, 0.00042, 0.00042, 0.00043, 0.00042, 0.00042, 0.00042, 0.00043, 0.00042, 0.00041, 0.00041, 0.00042, 0.00042, 0.00043, 0.00042, 0.00043, 0.00042, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00043, 0.00043, 0.00043, 0.0004, 0.00041, 0.00043, 0.00042, 0.00042, 0.00043, 0.00042, 0.00043, 0.00042, 0.00042, 0.00048, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00043, 0.00044, 0.00042, 0.00042, 0.00041, 0.0004, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00042, 0.00042, 0.00038, 0.0004, 0.00043, 0.00041, 0.00043, 0.00041, 0.0004, 0.0004, 0.0004, 0.00041, 0.00042, 0.00041, 0.00042, 0.00041, 0.00041, 0.0004, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00043, 0.00043, 0.00041, 0.00041, 0.00041, 0.00041, 0.00042, 0.00038, 0.0004, 0.00039, 0.00041, 0.00042, 0.00043, 0.00038, 0.00038, 0.0004, 0.00042, 0.0004, 0.0004, 0.0004, 0.00041, 0.00041, 0.0004, 0.00045, 0.00041, 0.00041, 0.0004, 0.00043, 0.00042, 0.00042, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00041, 0.00041, 0.00041, 0.00041, 0.0004, 0.00041, 0.00041, 0.0004, 0.00041, 0.00041, 0.0004, 0.00041, 0.00041, 0.0004, 0.00041, 0.00041, 0.0004, 0.00041, 0.00042, 0.00041, 0.0004, 0.00041, 0.00042, 0.00041, 0.00041, 0.0004, 0.00041, 0.0004, 0.00041, 0.00043, 0.0004, 0.00042, 0.00042, 0.00043, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00043, 0.00042, 0.00041, 0.00038, 0.00042, 0.00041, 0.00041, 0.00041, 0.00042, 0.00041, 0.00041, 0.00042, 0.00041, 0.0004, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00042, 0.00043, 0.00042, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00043, 0.00041, 0.0004, 0.00042, 0.00041, 0.00041, 0.00042, 0.00041, 0.00041, 0.00042, 0.00042, 0.0004, 0.00041, 0.00041, 0.00041, 0.00046, 0.00043, 0.00043, 0.00042, 0.00042, 0.00042, 0.00042, 0.00043, 0.00042, 0.00041, 0.00043, 0.00043, 0.00039, 0.00043, 0.00042, 0.00042, 0.00043, 0.00042, 0.00042, 0.00042, 0.00043, 0.0004, 0.00042, 0.0004, 0.00043, 0.00041, 0.00042, 0.00042, 0.00043, 0.00041, 0.00041, 0.00041, 0.00042, 0.00042, 0.00042, 0.00041, 0.00043, 0.00042, 0.0004, 0.00043, 0.00041, 0.00042, 0.00041, 0.00041, 0.00043, 0.00042, 0.00042, 0.00043, 0.00042, 0.00042, 0.00041, 0.00041, 0.00041, 0.00041, 0.00041, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00042, 0.00042, 0.00042, 0.00043, 0.00041, 0.00042, 0.00042, 0.00043, 0.00044, 0.00043, 0.00041, 0.00041, 0.00042, 0.00042, 0.00041, 0.00043, 0.00041, 0.00042, 0.00041, 0.00042, 0.00041, 0.00039, 0.00041, 0.00042, 0.00042, 0.00041, 0.00042, 0.00042, 0.00042, 0.00042, 0.00041, 0.00041, 0.00042, 0.00041, 0.00041, 0.00043, 0.00042, 0.00042, 0.00042, 0.00041, 0.00041, 0.00042, 0.00043, 0.00041, 0.00041, 0.00041, 0.00042, 0.00043, 0.00042, 0.00042, 0.00044, 0.00043, 0.00042, 0.00041, 0.00042, 0.00041, 0.00043, 0.00041, 0.00044, 0.0004, 0.00042, 0.00042, 0.00041, 0.00042, 0.00042, 0.00043, 0.00042, 0.00041, 0.00041, 0.00041, 0.00042, 0.00041, 0.00041, 0.00042, 0.00041, 0.0004, 0.00041, 0.00041, 0.00041, 0.00042, 0.00041, 0.0004, 0.00052, 0.00042, 0.00042, 0.00042, 0.0004, 0.00042, 0.00041, 0.00041]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02442, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00044, 0.00046, 0.00045, 0.00046, 0.00069, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00044, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.0005, 0.00046, 0.00045, 0.00044, 0.00047, 0.00046, 0.00045, 0.00053, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00046, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00044, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00052, 0.00045, 0.00047, 0.00046, 0.00039, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.0004, 0.00046, 0.00044, 0.0004, 0.00046, 0.00044, 0.0004, 0.0004, 0.0004, 0.00041, 0.00047, 0.00046, 0.0004, 0.00046, 0.00045, 0.00045, 0.00039, 0.00045, 0.00047, 0.00045, 0.0004, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00044, 0.00045, 0.00044, 0.00045, 0.00045, 0.00046, 0.00044, 0.00044, 0.00045, 0.00045, 0.00045, 0.00047, 0.00045, 0.00046, 0.00045, 0.00045, 0.00049, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00047, 0.00048, 0.00047, 0.00046, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00047, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00058, 0.00047, 0.00044, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00046, 0.00045, 0.00054, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00051, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00044, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00048, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00048, 0.00044, 0.00044, 0.00045, 0.00045, 0.00044, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00047, 0.00045, 0.00057, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00047, 0.00047, 0.00045, 0.00046, 0.00045, 0.00045, 0.00044, 0.00046, 0.00046, 0.00045, 0.00045, 0.00047, 0.00047, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00046, 0.00059, 0.00045, 0.00047, 0.00045, 0.00046, 0.00045, 0.00045, 0.00045]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00264, 0.00186, 0.00189, 0.00186, 0.00191, 0.00186, 0.00187, 0.00189, 0.0019, 0.00189, 0.00189, 0.002, 0.00187, 0.00201, 0.0019, 0.00186, 0.00187, 0.00185, 0.00187, 0.00187, 0.00186, 0.00186, 0.00187, 0.00186, 0.00187, 0.00189, 0.00189, 0.00185, 0.00188, 0.00186, 0.00187, 0.00188, 0.00188, 0.00186, 0.00188, 0.00187, 0.00189, 0.00185, 0.00189, 0.00189, 0.00187, 0.00186, 0.00186, 0.00189, 0.00188, 0.00186, 0.00186, 0.0019, 0.00186, 0.00187, 0.00188, 0.00186, 0.00213, 0.00189, 0.00185, 0.00186, 0.00188, 0.00189, 0.00186, 0.00185, 0.00187, 0.00186, 0.00186, 0.00186, 0.00186, 0.00186, 0.00185, 0.00186, 0.00187, 0.00186, 0.00186, 0.00189, 0.00188, 0.0019, 0.00189, 0.00187, 0.00187, 0.00188, 0.00186, 0.00187, 0.00187, 0.00188, 0.00186, 0.00186, 0.00186, 0.00185, 0.00186, 0.00186, 0.00187, 0.00186, 0.00217, 0.0019, 0.00195, 0.00188, 0.00187, 0.00188, 0.00188, 0.00186, 0.00188, 0.00186, 0.00188, 0.00188, 0.00186, 0.00187, 0.00188, 0.00185, 0.00208, 0.00187, 0.00187, 0.00186, 0.00185, 0.00185, 0.00188, 0.00185, 0.00186, 0.00186, 0.00186, 0.00186, 0.00186, 0.00186, 0.00187, 0.00185, 0.00185, 0.00188, 0.00186, 0.00185, 0.00188, 0.00186, 0.00186, 0.00184, 0.00187, 0.00186, 0.00189, 0.00186, 0.00185, 0.0019, 0.00187, 0.00186, 0.00186, 0.00186, 0.00186, 0.00186, 0.00189, 0.00187, 0.0019, 0.00186, 0.00186, 0.00187, 0.00188, 0.00185, 0.00186, 0.00186, 0.00189, 0.00186, 0.00187, 0.00187, 0.00203, 0.00186, 0.00186, 0.00188, 0.00187, 0.00186, 0.00188, 0.00184, 0.00185, 0.00186, 0.00187, 0.00185, 0.00186, 0.00187, 0.00188, 0.00198, 0.00198, 0.00186, 0.00185, 0.00187, 0.00188, 0.00186, 0.00188, 0.00185, 0.00185, 0.00187, 0.00187, 0.00186, 0.00185, 0.00185, 0.00187, 0.00186, 0.00186, 0.00187, 0.00187, 0.00185, 0.00187, 0.00187, 0.00186, 0.00185, 0.00186, 0.00187, 0.00188, 0.00191, 0.00186, 0.00188, 0.00188, 0.00187, 0.00188, 0.00187, 0.00188, 0.00186, 0.00187, 0.0019, 0.00187, 0.00187, 0.00186, 0.00187, 0.00187, 0.00186, 0.0019, 0.00188, 0.00187, 0.0019, 0.0019, 0.00191, 0.00191, 0.00186, 0.00187, 0.00188, 0.00187, 0.00186, 0.00188, 0.00188, 0.00189, 0.00189, 0.00188, 0.00188, 0.00189, 0.00189, 0.00189, 0.00186, 0.00191, 0.00189, 0.00187, 0.00186, 0.0019, 0.00188, 0.00188, 0.00187, 0.00188, 0.0019, 0.00189, 0.0019, 0.00219, 0.00189, 0.0019, 0.00187, 0.00188, 0.00187, 0.00187, 0.00188, 0.00188, 0.00187, 0.00186, 0.00189, 0.00188, 0.00188, 0.00188, 0.00188, 0.00188, 0.00189, 0.00188, 0.00216, 0.00188, 0.00189, 0.00188, 0.00189, 0.00189, 0.00189, 0.00187, 0.00187, 0.00188, 0.00188, 0.00199, 0.00187, 0.00201, 0.00189, 0.00187, 0.00191, 0.00189, 0.00187, 0.00188, 0.00188, 0.00189, 0.00246, 0.00272, 0.00189, 0.00189, 0.00189, 0.00288, 0.00189, 0.00187, 0.00189, 0.00189, 0.0019, 0.0019, 0.00188, 0.0019, 0.0019, 0.00191, 0.0019, 0.0019, 0.0019, 0.00191, 0.00191, 0.00189, 0.00189, 0.0019, 0.0019, 0.00189, 0.00188, 0.00188, 0.0019, 0.00197, 0.00187, 0.00189, 0.00188, 0.00189, 0.00187, 0.0019, 0.00187, 0.00189, 0.00188, 0.00189, 0.00188, 0.00187, 0.00187, 0.00188, 0.0019, 0.00187, 0.00188, 0.00188, 0.00188, 0.00191, 0.00216, 0.00186, 0.00188, 0.00189, 0.00189, 0.00187, 0.00189, 0.0019, 0.00187, 0.00189, 0.00187, 0.00199, 0.00189, 0.00188, 0.00187, 0.00187, 0.00188, 0.00189, 0.00188, 0.00188, 0.00188, 0.00188, 0.00187, 0.00188, 0.00188, 0.00188, 0.00189, 0.00188, 0.00188, 0.0019, 0.00187, 0.00189, 0.00189, 0.00188, 0.00189, 0.00188, 0.00188, 0.00188, 0.00189, 0.00186, 0.00189, 0.00187, 0.00189, 0.0019, 0.0019, 0.00194, 0.00189, 0.00187, 0.00187, 0.00189, 0.00189, 0.002, 0.00187, 0.00187, 0.00189, 0.00187, 0.00188, 0.00189, 0.00195]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00219, 0.00036, 0.00035, 0.00037, 0.00037, 0.00039, 0.00038, 0.00037, 0.00037, 0.00038, 0.00037, 0.0004, 0.00038, 0.00038, 0.00047, 0.00037, 0.00038, 0.00038, 0.00037, 0.00037, 0.00037, 0.00039, 0.00038, 0.00037, 0.00039, 0.00037, 0.00038, 0.00038, 0.00037, 0.00037, 0.00037, 0.00038, 0.00038, 0.00038, 0.00037, 0.00037, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00037, 0.00038, 0.00037, 0.00038, 0.00037, 0.00039, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00037, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00037, 0.00038, 0.0004, 0.00039, 0.0004, 0.00038, 0.00039, 0.00039, 0.00039, 0.00039, 0.00038, 0.00038, 0.00037, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00044, 0.00039, 0.0004, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00039, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.00037, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.0004, 0.00038, 0.00038, 0.00039, 0.00039, 0.0004, 0.00039, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00039, 0.00039, 0.00038, 0.00039, 0.00039, 0.00037, 0.00039, 0.00037, 0.00038, 0.00041, 0.00037, 0.00037, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.00038, 0.0004, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.0004, 0.00038, 0.0004, 0.00038, 0.00038, 0.0004, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.0004, 0.00038, 0.00038, 0.00038, 0.00038, 0.00037, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00037, 0.00038, 0.00038, 0.00037, 0.00038, 0.00038, 0.00037, 0.00037, 0.00039, 0.00038, 0.00038, 0.00037, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00037, 0.00037, 0.00038, 0.00038, 0.00043, 0.00037, 0.00038, 0.00038, 0.00037, 0.00038, 0.00038, 0.00038, 0.00038, 0.00037, 0.00038, 0.00037, 0.00037, 0.00038, 0.00037, 0.00039, 0.00037, 0.00037, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.0004, 0.0004, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.0004, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00039, 0.00038, 0.00037, 0.00038, 0.00039, 0.00039, 0.00038, 0.00037, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00037, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.0004, 0.00039, 0.00038, 0.00038, 0.00041, 0.0004, 0.00039, 0.00038, 0.0004, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00039, 0.00039, 0.00039, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.0004, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00039, 0.00039, 0.00038, 0.00039, 0.00038, 0.00039, 0.00038, 0.00039, 0.00038, 0.00041, 0.00039, 0.00039, 0.00041, 0.00038, 0.00038, 0.00052, 0.00038, 0.00039, 0.00038, 0.00038, 0.00038, 0.00038, 0.00038]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00097, 0.00085, 0.00083, 0.00104, 0.00084, 0.00083, 0.00084, 0.00085, 0.00085, 0.00084, 0.00083, 0.00085, 0.00083, 0.00085, 0.00178, 0.00084, 0.00084, 0.00084, 0.00084, 0.00084, 0.00085, 0.00085, 0.00085, 0.00083, 0.00082, 0.00083, 0.00084, 0.00083, 0.00084, 0.00084, 0.00083, 0.00084, 0.00083, 0.00086, 0.00085, 0.00085, 0.00084, 0.00084, 0.00085, 0.00085, 0.00084, 0.00084, 0.00084, 0.00085, 0.00085, 0.00084, 0.00085, 0.00085, 0.00084, 0.00085, 0.00118, 0.00086, 0.00087, 0.00086, 0.00108, 0.00085, 0.00085, 0.00084, 0.00084, 0.00083, 0.00085, 0.00109, 0.00084, 0.00083, 0.00084, 0.00086, 0.00085, 0.00086, 0.00085, 0.00085, 0.00085, 0.00086, 0.00085, 0.00084, 0.00087, 0.00085, 0.00087, 0.00084, 0.00086, 0.00085, 0.00085, 0.00084, 0.00085, 0.00084, 0.00085, 0.00084, 0.00085, 0.00087, 0.00085, 0.00087, 0.00096, 0.00085, 0.00085, 0.00086, 0.00084, 0.00085, 0.00086, 0.00083, 0.00085, 0.00084, 0.00083, 0.00084, 0.00084, 0.00083, 0.00084, 0.00084, 0.00085, 0.00083, 0.00083, 0.00083, 0.00083, 0.00084, 0.00083, 0.00084, 0.00083, 0.00083, 0.00085, 0.00084, 0.00083, 0.00084, 0.00083, 0.00084, 0.00083, 0.00084, 0.00084, 0.00083, 0.00084, 0.00086, 0.00084, 0.00083, 0.00084, 0.00084, 0.00083, 0.00085, 0.00085, 0.00084, 0.00083, 0.00086, 0.00086, 0.00084, 0.00085, 0.00083, 0.00084, 0.00084, 0.00083, 0.00084, 0.00083, 0.00083, 0.00083, 0.00084, 0.00085, 0.00085, 0.00083, 0.00084, 0.00083, 0.00083, 0.00094, 0.00084, 0.00084, 0.00083, 0.00084, 0.00084, 0.00083, 0.00085, 0.00084, 0.00084, 0.00083, 0.00085, 0.00084, 0.00084, 0.00083, 0.00085, 0.00084, 0.00083, 0.00085, 0.00083, 0.00083, 0.00085, 0.00083, 0.00084, 0.00098, 0.00085, 0.00084, 0.00085, 0.00083, 0.00083, 0.00084, 0.00085, 0.00085, 0.00085, 0.00084, 0.00084, 0.00084, 0.00084, 0.00083, 0.00085, 0.00085, 0.00084, 0.00087, 0.00084, 0.00083, 0.00084, 0.00085, 0.00084, 0.00084, 0.00084, 0.00085, 0.00086, 0.00086, 0.00083, 0.00083, 0.00083, 0.00085, 0.00084, 0.00085, 0.00084, 0.00084, 0.00085, 0.00084, 0.00084, 0.00084, 0.00084, 0.00083, 0.00082, 0.00084, 0.00109, 0.00084, 0.00084, 0.00084, 0.00084, 0.00084, 0.00083, 0.00083, 0.00085, 0.00085, 0.00084, 0.00084, 0.00085, 0.00084, 0.00085, 0.00083, 0.00085, 0.00084, 0.00084, 0.00084, 0.00085, 0.00085, 0.00085, 0.00084, 0.00083, 0.00093, 0.00084, 0.00083, 0.00085, 0.00084, 0.00084, 0.00084, 0.00084, 0.00084, 0.00085, 0.00085, 0.00083, 0.00084, 0.00084, 0.00083, 0.00085, 0.00086, 0.00085, 0.00083, 0.00085, 0.00085, 0.00084, 0.00085, 0.00084, 0.00084, 0.00085, 0.00085, 0.00085, 0.00084, 0.00085, 0.00083, 0.00084, 0.00083, 0.00084, 0.00085, 0.00083, 0.00084, 0.00086, 0.00086, 0.00085, 0.00084, 0.00102, 0.00089, 0.00085, 0.00084, 0.00084, 0.00084, 0.00084, 0.00085, 0.00085, 0.00084, 0.00086, 0.00096, 0.00083, 0.00085, 0.00084, 0.00084, 0.00085, 0.00084, 0.00084, 0.00083, 0.00085, 0.00084, 0.00085, 0.00085, 0.00083, 0.00084, 0.00085, 0.00085, 0.00084, 0.00086, 0.00084, 0.00084, 0.00083, 0.00095, 0.00084, 0.00084, 0.00086, 0.00085, 0.00084, 0.00085, 0.00084, 0.00084, 0.00086, 0.00085, 0.00085, 0.00085, 0.00084, 0.00083, 0.00087, 0.00084, 0.00093, 0.00085, 0.00084, 0.00084, 0.00085, 0.00083, 0.00083, 0.00084, 0.00083, 0.00085, 0.00086, 0.00084, 0.00113, 0.00084, 0.00083, 0.00084, 0.00103, 0.00085, 0.00084, 0.00087, 0.00084, 0.00084, 0.00084, 0.00083, 0.00084, 0.00086, 0.00084, 0.00084, 0.00082, 0.00085, 0.00085, 0.00083, 0.00084, 0.00084, 0.00084, 0.00084, 0.00085, 0.00084, 0.00084, 0.00082, 0.00085, 0.00084, 0.00083, 0.00084, 0.00085, 0.00094, 0.00085, 0.00085, 0.00086, 0.00116, 0.00084, 0.00137, 0.00084, 0.00083, 0.00084, 0.00084, 0.00104, 0.00085, 0.00083]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.03257, 0.00561, 0.00555, 0.00673, 0.00567, 0.00562, 0.00561, 0.00563, 0.00577, 0.00565, 0.00561, 0.00611, 0.00562, 0.00577, 0.00929, 0.00564, 0.00561, 0.00562, 0.0056, 0.00562, 0.0056, 0.00563, 0.00563, 0.00561, 0.00559, 0.00561, 0.00563, 0.00561, 0.00562, 0.00557, 0.0056, 0.00562, 0.00562, 0.00563, 0.00562, 0.00562, 0.00568, 0.00562, 0.00565, 0.00566, 0.00566, 0.00565, 0.0056, 0.00567, 0.00567, 0.00569, 0.00566, 0.00568, 0.00565, 0.00563, 0.00698, 0.00565, 0.00598, 0.0057, 0.00701, 0.00568, 0.00567, 0.00565, 0.00567, 0.00568, 0.00563, 0.00767, 0.00563, 0.00608, 0.00566, 0.00565, 0.00568, 0.00565, 0.00565, 0.00567, 0.00566, 0.00571, 0.00568, 0.00567, 0.00567, 0.00565, 0.00569, 0.00575, 0.00565, 0.00565, 0.00562, 0.00577, 0.00568, 0.00567, 0.00563, 0.00564, 0.00565, 0.0057, 0.00565, 0.00567, 0.00638, 0.00578, 0.00578, 0.00572, 0.0056, 0.00567, 0.00571, 0.00565, 0.00565, 0.00567, 0.00563, 0.00563, 0.00563, 0.00563, 0.00562, 0.00635, 0.00583, 0.00568, 0.00584, 0.00555, 0.00577, 0.00559, 0.0056, 0.00558, 0.00584, 0.00561, 0.00557, 0.00564, 0.00562, 0.00566, 0.00555, 0.00562, 0.00565, 0.00566, 0.00559, 0.0056, 0.00561, 0.00566, 0.00564, 0.00561, 0.00563, 0.00564, 0.00564, 0.00565, 0.00564, 0.00568, 0.00564, 0.00565, 0.00566, 0.00568, 0.00554, 0.00562, 0.00556, 0.00562, 0.0057, 0.00565, 0.00583, 0.00554, 0.00562, 0.00561, 0.00564, 0.00571, 0.00563, 0.00563, 0.00565, 0.0056, 0.00607, 0.00565, 0.00564, 0.00564, 0.00565, 0.00565, 0.00563, 0.00564, 0.00563, 0.00566, 0.00564, 0.00565, 0.00565, 0.00567, 0.00565, 0.00576, 0.00575, 0.00563, 0.00566, 0.00658, 0.00565, 0.00564, 0.00568, 0.00562, 0.00663, 0.00565, 0.00564, 0.00564, 0.00562, 0.00563, 0.00568, 0.00566, 0.00565, 0.00564, 0.00565, 0.00563, 0.00565, 0.00561, 0.00564, 0.00563, 0.00562, 0.00564, 0.00568, 0.00568, 0.00567, 0.00567, 0.00569, 0.00566, 0.0056, 0.00564, 0.00567, 0.00567, 0.00586, 0.00568, 0.00555, 0.00567, 0.00562, 0.00558, 0.00585, 0.00563, 0.00566, 0.00565, 0.00565, 0.00566, 0.00559, 0.00566, 0.00566, 0.00561, 0.00573, 0.00721, 0.00562, 0.00564, 0.00593, 0.00595, 0.00563, 0.00564, 0.00566, 0.00567, 0.00565, 0.00569, 0.00564, 0.00566, 0.00568, 0.00566, 0.00578, 0.00588, 0.0064, 0.00571, 0.00566, 0.00564, 0.00565, 0.00567, 0.00566, 0.00564, 0.00643, 0.00566, 0.00567, 0.00564, 0.00601, 0.00563, 0.00566, 0.00566, 0.00566, 0.00563, 0.00566, 0.00565, 0.00557, 0.00567, 0.00564, 0.00566, 0.00565, 0.00566, 0.00564, 0.00596, 0.00567, 0.00562, 0.00565, 0.00566, 0.00564, 0.00564, 0.00569, 0.00568, 0.00569, 0.00569, 0.00575, 0.00567, 0.00583, 0.00568, 0.00566, 0.00566, 0.00567, 0.00566, 0.00567, 0.00566, 0.00564, 0.00689, 0.00665, 0.00563, 0.00566, 0.00566, 0.00685, 0.00566, 0.00565, 0.00567, 0.00567, 0.00574, 0.00611, 0.00563, 0.00565, 0.00569, 0.00568, 0.00568, 0.00568, 0.0057, 0.00566, 0.00569, 0.00567, 0.0057, 0.00566, 0.00569, 0.00564, 0.00565, 0.00568, 0.00569, 0.00571, 0.00564, 0.00566, 0.00565, 0.0058, 0.00566, 0.00565, 0.00564, 0.00566, 0.00566, 0.00567, 0.00556, 0.00565, 0.00568, 0.00564, 0.00567, 0.00566, 0.00566, 0.00566, 0.00566, 0.00565, 0.00622, 0.00564, 0.00563, 0.00565, 0.0058, 0.00565, 0.00563, 0.00567, 0.00564, 0.00566, 0.00569, 0.00579, 0.0071, 0.00625, 0.00661, 0.00596, 0.00708, 0.00571, 0.00566, 0.00572, 0.0057, 0.00565, 0.00566, 0.00568, 0.00566, 0.00569, 0.00565, 0.00568, 0.00558, 0.00572, 0.00566, 0.00564, 0.00571, 0.00569, 0.00569, 0.00567, 0.00567, 0.00564, 0.00569, 0.00563, 0.0057, 0.00565, 0.00567, 0.00569, 0.00565, 0.00602, 0.00567, 0.00566, 0.00568, 0.00691, 0.00568, 0.00824, 0.00567, 0.00569, 0.00565, 0.00566, 0.00689, 0.00567, 0.00569]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84303, 10.86032, 10.84988, 10.84755, 10.76639, 10.77411, 10.67857, 10.53004, 10.38397, 10.29666, 9.92036, 10.03609, 10.04286, 9.75368, 9.87024, 9.57458, 9.50956, 9.70645, 9.43156, 9.37511, 9.284, 9.18283, 9.20684, 9.02346, 9.21677, 9.08417, 9.17277, 9.18323, 9.31569, 9.00474, 8.94547, 9.06044, 9.05792, 8.66708, 8.73014, 8.76017, 8.69512, 8.74237, 8.66438, 8.77103, 8.66577, 8.85394, 8.83642, 8.49824, 8.38764, 8.42876, 8.48638, 8.38112, 8.42721, 8.57916, 8.36213, 8.18555, 8.21868, 8.21376, 8.25912, 7.90597, 8.08558, 7.88018, 8.23297, 8.21565, 7.99013, 7.95413, 7.90374, 7.72213, 7.72557, 7.62784, 7.49843, 7.88783, 7.68211, 7.43256, 7.72606, 7.75519, 7.5254, 7.28466, 7.43748, 7.32478, 7.44941, 7.21198, 7.61949, 7.26498, 7.33394, 7.19595, 7.19608, 7.40347, 7.15606, 7.26585, 6.98127, 6.98967, 7.02701, 7.12404, 6.81114, 6.9732, 7.07844, 6.98715, 6.86379, 6.74535, 6.97969, 7.04992, 6.69473, 6.57332, 6.71755, 6.73627, 6.72482, 6.72951, 6.64965, 6.39869, 6.62934, 6.6128, 6.44062, 6.62092, 6.73782, 6.60642, 6.72099, 6.69098, 6.62325, 6.50501, 6.59411, 6.40344, 6.66286, 6.24475, 6.24827, 6.29959, 6.38833, 6.34649, 6.44604, 6.28662, 6.33306, 6.23143, 6.1945, 6.39075, 6.31833, 6.31606, 6.15661, 6.15059, 6.23078, 6.37677, 6.19418, 6.14556, 6.174, 6.10964, 6.05825, 6.06794, 6.25281, 6.40554, 6.25551, 6.29757, 6.09544, 6.1725, 6.00218, 6.02712, 5.95524, 6.25067, 6.1861, 5.96596, 5.78395, 6.12333, 5.84793, 6.10088, 5.78605, 6.16305, 6.14324, 6.08193, 5.9272, 6.11128, 5.94147, 6.19288, 5.88909, 5.78652, 5.77759, 5.68182, 6.00901, 5.99171, 6.064, 5.887, 6.03556, 5.96156, 5.98678, 5.98309, 5.94332, 5.83241, 5.94309, 5.60951, 5.69435, 5.88169, 5.83567, 5.85447, 5.75902, 5.83004, 5.71739, 5.55081, 5.71567, 5.61507, 5.82158, 5.59427, 5.70169, 5.70024, 5.89399, 5.63586, 5.84189, 5.73395, 5.86128, 5.31906, 5.89065, 5.8668, 5.84568, 5.40705, 5.40162, 5.61805, 5.58944, 5.47887, 5.57169, 5.66894, 5.46961, 5.737, 5.50292, 5.58399, 5.61697, 5.61602, 5.50714, 5.6077, 5.6651, 5.67541, 5.58049, 5.65548, 5.36443, 5.67256, 5.62445, 5.41886, 5.57712, 5.62171, 5.55213, 5.34421, 5.53498, 5.48095, 5.4778, 5.37859, 5.55337, 5.60077, 5.38946, 5.5161, 5.4845, 5.3308, 5.503, 5.40661, 5.44202, 5.3156, 5.06608, 5.47488, 5.56633, 5.71203, 5.41237, 5.602, 5.6336, 5.23514, 5.26957, 5.38908, 5.39646, 5.32832, 5.49536, 5.18302, 5.2973, 5.24699, 5.3738, 5.2533, 5.4419, 5.53407, 5.31248, 5.43315, 5.33688, 5.07446, 5.3117, 5.25312, 5.30184, 5.11129, 5.27552, 5.26324, 5.47224, 5.15822, 5.26777, 5.21213, 5.35617, 4.98409, 4.9122, 5.32204, 5.39135, 5.22909, 5.3223, 5.10207, 5.16342, 5.26324, 5.06816, 5.26642, 5.06638, 5.34472, 5.24739, 5.15433, 5.24748, 5.04399, 5.32024, 5.05488, 5.02871, 5.1457, 5.11299, 5.27264, 5.15675, 5.28106, 5.09695, 5.09458, 5.25141, 5.32789, 5.25804, 5.19731, 5.14154, 5.29133, 4.95279, 5.2099, 5.09154, 5.30528, 5.17547, 5.19246, 5.11436, 4.986, 4.99619, 5.22741, 5.31255, 5.10417, 5.06172, 4.91443, 5.12691, 5.1217, 4.93205, 5.34318, 5.02802, 5.10574, 5.17142, 5.00778, 5.07028, 5.0728, 4.99912, 5.08403, 5.16803, 4.98253, 5.18553, 4.93609, 4.93034, 5.06451, 5.00328, 4.9143, 4.78254, 4.9515, 5.1248, 5.02128, 5.01937, 5.34246, 4.96515, 4.99654, 5.05289, 4.816, 4.74072, 4.99878, 5.04752, 4.87941, 4.96151, 5.05319, 5.02704, 4.8254, 4.8992, 4.91046, 4.83957, 4.74493, 5.01861, 4.76013, 5.21014, 4.79858, 5.00113, 4.74548, 4.79219, 4.82659, 4.65777, 4.66208, 4.84897, 4.81474, 4.80913, 4.92799, 4.89236, 4.93339, 4.77993, 4.89168, 4.7432, 4.92229, 4.96619, 4.88011, 4.71273, 4.7931, 4.91139, 4.72229, 4.87421, 4.70468, 4.69956, 4.65227]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.84303, 10.86032, 10.84988, 10.84755, 10.76639, 10.77411, 10.67857, 10.53004, 10.38397, 10.29666, 9.92036, 10.03609, 10.04286, 9.75368, 9.87024, 9.57458, 9.50956, 9.70645, 9.43156, 9.37511, 9.284, 9.18283, 9.20684, 9.02346, 9.21677, 9.08417, 9.17277, 9.18323, 9.31569, 9.00474, 8.94547, 9.06044, 9.05792, 8.66708, 8.73014, 8.76017, 8.69512, 8.74237, 8.66438, 8.77103, 8.66577, 8.85394, 8.83642, 8.49824, 8.38764, 8.42876, 8.48638, 8.38112, 8.42721, 8.57916, 8.36213, 8.18555, 8.21868, 8.21376, 8.25912, 7.90597, 8.08558, 7.88018, 8.23297, 8.21565, 7.99013, 7.95413, 7.90374, 7.72213, 7.72557, 7.62784, 7.49843, 7.88783, 7.68211, 7.43256, 7.72606, 7.75519, 7.5254, 7.28466, 7.43748, 7.32478, 7.44941, 7.21198, 7.61949, 7.26498, 7.33394, 7.19595, 7.19608, 7.40347, 7.15606, 7.26585, 6.98127, 6.98967, 7.02701, 7.12404, 6.81114, 6.9732, 7.07844, 6.98715, 6.86379, 6.74535, 6.97969, 7.04992, 6.69473, 6.57332, 6.71755, 6.73627, 6.72482, 6.72951, 6.64965, 6.39869, 6.62934, 6.6128, 6.44062, 6.62092, 6.73782, 6.60642, 6.72099, 6.69098, 6.62325, 6.50501, 6.59411, 6.40344, 6.66286, 6.24475, 6.24827, 6.29959, 6.38833, 6.34649, 6.44604, 6.28662, 6.33306, 6.23143, 6.1945, 6.39075, 6.31833, 6.31606, 6.15661, 6.15059, 6.23078, 6.37677, 6.19418, 6.14556, 6.174, 6.10964, 6.05825, 6.06794, 6.25281, 6.40554, 6.25551, 6.29757, 6.09544, 6.1725, 6.00218, 6.02712, 5.95524, 6.25067, 6.1861, 5.96596, 5.78395, 6.12333, 5.84793, 6.10088, 5.78605, 6.16305, 6.14324, 6.08193, 5.9272, 6.11128, 5.94147, 6.19288, 5.88909, 5.78652, 5.77759, 5.68182, 6.00901, 5.99171, 6.064, 5.887, 6.03556, 5.96156, 5.98678, 5.98309, 5.94332, 5.83241, 5.94309, 5.60951, 5.69435, 5.88169, 5.83567, 5.85447, 5.75902, 5.83004, 5.71739, 5.55081, 5.71567, 5.61507, 5.82158, 5.59427, 5.70169, 5.70024, 5.89399, 5.63586, 5.84189, 5.73395, 5.86128, 5.31906, 5.89065, 5.8668, 5.84568, 5.40705, 5.40162, 5.61805, 5.58944, 5.47887, 5.57169, 5.66894, 5.46961, 5.737, 5.50292, 5.58399, 5.61697, 5.61602, 5.50714, 5.6077, 5.6651, 5.67541, 5.58049, 5.65548, 5.36443, 5.67256, 5.62445, 5.41886, 5.57712, 5.62171, 5.55213, 5.34421, 5.53498, 5.48095, 5.4778, 5.37859, 5.55337, 5.60077, 5.38946, 5.5161, 5.4845, 5.3308, 5.503, 5.40661, 5.44202, 5.3156, 5.06608, 5.47488, 5.56633, 5.71203, 5.41237, 5.602, 5.6336, 5.23514, 5.26957, 5.38908, 5.39646, 5.32832, 5.49536, 5.18302, 5.2973, 5.24699, 5.3738, 5.2533, 5.4419, 5.53407, 5.31248, 5.43315, 5.33688, 5.07446, 5.3117, 5.25312, 5.30184, 5.11129, 5.27552, 5.26324, 5.47224, 5.15822, 5.26777, 5.21213, 5.35617, 4.98409, 4.9122, 5.32204, 5.39135, 5.22909, 5.3223, 5.10207, 5.16342, 5.26324, 5.06816, 5.26642, 5.06638, 5.34472, 5.24739, 5.15433, 5.24748, 5.04399, 5.32024, 5.05488, 5.02871, 5.1457, 5.11299, 5.27264, 5.15675, 5.28106, 5.09695, 5.09458, 5.25141, 5.32789, 5.25804, 5.19731, 5.14154, 5.29133, 4.95279, 5.2099, 5.09154, 5.30528, 5.17547, 5.19246, 5.11436, 4.986, 4.99619, 5.22741, 5.31255, 5.10417, 5.06172, 4.91443, 5.12691, 5.1217, 4.93205, 5.34318, 5.02802, 5.10574, 5.17142, 5.00778, 5.07028, 5.0728, 4.99912, 5.08403, 5.16803, 4.98253, 5.18553, 4.93609, 4.93034, 5.06451, 5.00328, 4.9143, 4.78254, 4.9515, 5.1248, 5.02128, 5.01937, 5.34246, 4.96515, 4.99654, 5.05289, 4.816, 4.74072, 4.99878, 5.04752, 4.87941, 4.96151, 5.05319, 5.02704, 4.8254, 4.8992, 4.91046, 4.83957, 4.74493, 5.01861, 4.76013, 5.21014, 4.79858, 5.00113, 4.74548, 4.79219, 4.82659, 4.65777, 4.66208, 4.84897, 4.81474, 4.80913, 4.92799, 4.89236, 4.93339, 4.77993, 4.89168, 4.7432, 4.92229, 4.96619, 4.88011, 4.71273, 4.7931, 4.91139, 4.72229, 4.87421, 4.70468, 4.69956, 4.65227]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.92196, 13.64105, 14.19575, 13.10329, 13.56093, 11.06924, 10.32704, 12.58903, 11.89406, 9.6749, 7.04626, 4.0336, 3.15187, 2.82418, 2.35804, 2.43442, 2.16004, 1.97461, 2.14035, 2.12249, 2.20138, 2.2657, 2.05671, 2.22896, 1.95829, 2.02503, 1.88632, 1.84693, 1.87101, 2.18322, 2.10962, 1.97689, 1.94956, 2.15482, 2.33059, 2.0713, 2.06596, 1.83468, 1.98146, 1.78906, 2.08095, 1.74031, 1.73584, 1.83223, 1.93635, 1.78517, 1.74533, 1.74989, 1.72773, 1.51419, 1.74951, 1.76214, 1.76755, 1.83739, 1.54724, 1.80208, 1.67454, 1.80868, 1.51645, 1.42949, 1.65422, 1.43167, 1.74384, 1.82674, 1.56795, 1.61973, 1.62231, 1.51322, 1.4269, 1.55439, 1.3649, 1.40671, 1.47679, 1.40979, 1.35488, 1.43798, 1.41114, 1.34745, 1.32431, 1.23395, 1.36576, 1.22914, 1.25372, 1.35028, 1.23455, 1.29297, 1.37717, 1.26373, 1.37004, 1.08995, 1.10379, 1.10875, 1.15108, 1.26523, 0.89985, 1.39001, 1.10735, 1.30884, 1.00577, 1.31705, 1.15922, 1.16049, 1.08293, 1.30514, 0.98385, 1.11074, 1.1592, 0.9745, 1.26156, 1.13226, 0.98984, 0.97441, 0.96023, 0.94898, 1.04337, 1.04095, 0.96044, 1.19634, 1.26146, 1.4137, 0.97849, 1.01274, 1.06643, 1.01496, 0.94459, 1.13752, 1.02579, 1.05074, 1.22247, 1.26548, 1.04774, 1.44863, 1.15549, 1.15597, 1.19734, 1.2287, 1.25743, 1.88802, 1.76897, 1.48112, 1.4651, 1.39709, 1.38654, 1.09404, 1.62425, 1.69258, 1.31425, 1.11912, 1.16099, 1.18343, 1.29282, 1.58176, 1.59702, 1.35711, 1.25116, 1.93028, 1.26411, 1.16234, 1.73045, 1.37516, 1.21056, 1.1698, 1.36362, 1.31019, 1.41174, 1.1141, 1.35444, 1.27655, 1.56101, 1.26438, 1.09582, 1.27416, 1.41508, 1.54422, 1.36323, 1.24407, 1.29014, 1.18935, 1.13176, 1.03122, 1.33001, 1.37077, 1.14753, 1.11258, 1.66325, 1.11887, 1.76805, 1.40233, 1.37783, 1.50291, 1.27142, 1.30216, 1.29887, 1.46138, 1.55382, 1.23876, 1.8076, 1.40113, 1.63396, 1.55057, 1.08699, 1.24471, 1.22211, 1.14251, 1.26485, 1.45246, 1.55789, 1.71804, 1.37054, 1.61527, 1.57346, 1.43675, 1.26103, 1.17063, 1.56904, 1.17977, 1.4408, 1.72049, 1.50941, 1.30391, 1.34373, 1.32377, 1.27909, 1.56247, 1.31671, 1.38601, 1.61151, 1.49478, 1.75857, 1.27914, 1.31454, 2.08285, 1.65152, 1.54337, 1.46369, 1.68505, 1.74708, 1.34813, 1.53151, 1.36655, 1.5068, 1.33926, 1.42092, 1.39573, 1.3088, 1.90711, 1.46652, 1.29613, 1.44842, 1.30354, 1.28453, 1.49548, 1.47812, 1.39914, 1.32083, 1.19715, 1.79989, 1.43253, 1.35222, 1.42532, 1.23793, 1.41904, 1.21814, 1.25683, 1.2335, 1.46238, 1.48727, 1.4808, 1.33354, 1.33662, 1.26457, 1.31807, 1.46217, 1.35853, 1.55295, 1.20988, 1.50233, 1.51611, 1.48328, 1.32591, 1.35903, 1.25739, 1.45462, 1.40772, 1.52784, 1.49325, 1.48176, 1.41498, 1.37099, 1.4565, 1.35995, 1.85538, 1.22436, 1.50223, 1.62834, 2.02006, 1.60123, 1.72187, 1.44841, 1.22003, 1.2907, 1.31733, 1.13053, 1.33575, 1.57284, 1.47894, 1.41277, 1.40064, 1.30099, 1.35607, 1.52515, 1.48522, 1.31187, 1.24496, 1.36995, 1.60389, 1.24009, 1.55027, 1.2329, 1.34795, 1.32343, 1.38946, 1.27338, 1.46297, 1.50613, 1.56272, 1.67908, 1.41893, 1.40655, 1.34016, 1.79612, 1.52344, 1.31538, 1.82889, 1.5317, 1.18989, 1.44241, 1.33335, 1.49631, 1.45109, 1.41567, 1.28181, 1.28831, 1.39113, 1.42151, 1.1475, 1.49249, 1.42727, 1.4635, 1.13088, 1.41, 1.30719, 1.30003, 1.92172, 1.44667, 1.42061, 1.31137, 1.5365, 1.46596, 1.30019, 1.53226, 1.21709, 1.36071, 1.47588, 1.10067, 1.46261, 1.69979, 1.33386, 1.3067, 1.50275, 1.48945, 1.4021, 1.56615, 1.59437, 1.41693, 1.52987, 1.27517, 1.55287, 1.38137, 1.28009, 1.33198, 1.29291, 1.40497, 1.25603, 1.18811, 1.37138, 1.43758, 1.46419, 1.4718, 1.35085, 1.22463, 1.2576, 1.44724, 1.32087, 1.61352, 1.4648, 1.47154, 1.80709, 1.41366, 1.12723]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.92196, 13.64105, 14.19575, 13.10329, 13.56093, 11.06924, 10.32704, 12.58903, 11.89406, 9.6749, 7.04626, 4.0336, 3.15187, 2.82418, 2.35804, 2.43442, 2.16004, 1.97461, 2.14035, 2.12249, 2.20138, 2.2657, 2.05671, 2.22896, 1.95829, 2.02503, 1.88632, 1.84693, 1.87101, 2.18322, 2.10962, 1.97689, 1.94956, 2.15482, 2.33059, 2.0713, 2.06596, 1.83468, 1.98146, 1.78906, 2.08095, 1.74031, 1.73584, 1.83223, 1.93635, 1.78517, 1.74533, 1.74989, 1.72773, 1.51419, 1.74951, 1.76214, 1.76755, 1.83739, 1.54724, 1.80208, 1.67454, 1.80868, 1.51645, 1.42949, 1.65422, 1.43167, 1.74384, 1.82674, 1.56795, 1.61973, 1.62231, 1.51322, 1.4269, 1.55439, 1.3649, 1.40671, 1.47679, 1.40979, 1.35488, 1.43798, 1.41114, 1.34745, 1.32431, 1.23395, 1.36576, 1.22914, 1.25372, 1.35028, 1.23455, 1.29297, 1.37717, 1.26373, 1.37004, 1.08995, 1.10379, 1.10875, 1.15108, 1.26523, 0.89985, 1.39001, 1.10735, 1.30884, 1.00577, 1.31705, 1.15922, 1.16049, 1.08293, 1.30514, 0.98385, 1.11074, 1.1592, 0.9745, 1.26156, 1.13226, 0.98984, 0.97441, 0.96023, 0.94898, 1.04337, 1.04095, 0.96044, 1.19634, 1.26146, 1.4137, 0.97849, 1.01274, 1.06643, 1.01496, 0.94459, 1.13752, 1.02579, 1.05074, 1.22247, 1.26548, 1.04774, 1.44863, 1.15549, 1.15597, 1.19734, 1.2287, 1.25743, 1.88802, 1.76897, 1.48112, 1.4651, 1.39709, 1.38654, 1.09404, 1.62425, 1.69258, 1.31425, 1.11912, 1.16099, 1.18343, 1.29282, 1.58176, 1.59702, 1.35711, 1.25116, 1.93028, 1.26411, 1.16234, 1.73045, 1.37516, 1.21056, 1.1698, 1.36362, 1.31019, 1.41174, 1.1141, 1.35444, 1.27655, 1.56101, 1.26438, 1.09582, 1.27416, 1.41508, 1.54422, 1.36323, 1.24407, 1.29014, 1.18935, 1.13176, 1.03122, 1.33001, 1.37077, 1.14753, 1.11258, 1.66325, 1.11887, 1.76805, 1.40233, 1.37783, 1.50291, 1.27142, 1.30216, 1.29887, 1.46138, 1.55382, 1.23876, 1.8076, 1.40113, 1.63396, 1.55057, 1.08699, 1.24471, 1.22211, 1.14251, 1.26485, 1.45246, 1.55789, 1.71804, 1.37054, 1.61527, 1.57346, 1.43675, 1.26103, 1.17063, 1.56904, 1.17977, 1.4408, 1.72049, 1.50941, 1.30391, 1.34373, 1.32377, 1.27909, 1.56247, 1.31671, 1.38601, 1.61151, 1.49478, 1.75857, 1.27914, 1.31454, 2.08285, 1.65152, 1.54337, 1.46369, 1.68505, 1.74708, 1.34813, 1.53151, 1.36655, 1.5068, 1.33926, 1.42092, 1.39573, 1.3088, 1.90711, 1.46652, 1.29613, 1.44842, 1.30354, 1.28453, 1.49548, 1.47812, 1.39914, 1.32083, 1.19715, 1.79989, 1.43253, 1.35222, 1.42532, 1.23793, 1.41904, 1.21814, 1.25683, 1.2335, 1.46238, 1.48727, 1.4808, 1.33354, 1.33662, 1.26457, 1.31807, 1.46217, 1.35853, 1.55295, 1.20988, 1.50233, 1.51611, 1.48328, 1.32591, 1.35903, 1.25739, 1.45462, 1.40772, 1.52784, 1.49325, 1.48176, 1.41498, 1.37099, 1.4565, 1.35995, 1.85538, 1.22436, 1.50223, 1.62834, 2.02006, 1.60123, 1.72187, 1.44841, 1.22003, 1.2907, 1.31733, 1.13053, 1.33575, 1.57284, 1.47894, 1.41277, 1.40064, 1.30099, 1.35607, 1.52515, 1.48522, 1.31187, 1.24496, 1.36995, 1.60389, 1.24009, 1.55027, 1.2329, 1.34795, 1.32343, 1.38946, 1.27338, 1.46297, 1.50613, 1.56272, 1.67908, 1.41893, 1.40655, 1.34016, 1.79612, 1.52344, 1.31538, 1.82889, 1.5317, 1.18989, 1.44241, 1.33335, 1.49631, 1.45109, 1.41567, 1.28181, 1.28831, 1.39113, 1.42151, 1.1475, 1.49249, 1.42727, 1.4635, 1.13088, 1.41, 1.30719, 1.30003, 1.92172, 1.44667, 1.42061, 1.31137, 1.5365, 1.46596, 1.30019, 1.53226, 1.21709, 1.36071, 1.47588, 1.10067, 1.46261, 1.69979, 1.33386, 1.3067, 1.50275, 1.48945, 1.4021, 1.56615, 1.59437, 1.41693, 1.52987, 1.27517, 1.55287, 1.38137, 1.28009, 1.33198, 1.29291, 1.40497, 1.25603, 1.18811, 1.37138, 1.43758, 1.46419, 1.4718, 1.35085, 1.22463, 1.2576, 1.44724, 1.32087, 1.61352, 1.4648, 1.47154, 1.80709, 1.41366, 1.12723]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [65.0, 71.0, 74.0, 78.0, 68.0, 65.0, 79.0, 104.0, 95.0, 118.0, 116.0, 161.0, 141.0, 148.0, 182.0, 146.0, 164.0, 199.0, 174.0, 205.0, 166.0, 167.0, 186.0, 158.0, 195.0, 179.0, 188.0, 208.0, 187.0, 145.0, 145.0, 146.0, 156.0, 175.0, 132.0, 180.0, 177.0, 205.0, 172.0, 159.0, 158.0, 175.0, 153.0, 203.0, 196.0, 170.0, 185.0, 179.0, 140.0, 227.0, 198.0, 165.0, 172.0, 149.0, 199.0, 213.0, 179.0, 157.0, 255.0, 240.0, 186.0, 191.0, 164.0, 186.0, 208.0, 229.0, 213.0, 198.0, 198.0, 178.0, 246.0, 222.0, 177.0, 236.0, 193.0, 215.0, 226.0, 205.0, 251.0, 226.0, 224.0, 245.0, 219.0, 205.0, 198.0, 190.0, 171.0, 191.0, 171.0, 187.0, 182.0, 207.0, 233.0, 201.0, 220.0, 152.0, 216.0, 194.0, 175.0, 157.0, 165.0, 188.0, 163.0, 163.0, 160.0, 155.0, 160.0, 167.0, 144.0, 190.0, 194.0, 143.0, 153.0, 175.0, 158.0, 147.0, 166.0, 115.0, 142.0, 141.0, 117.0, 131.0, 132.0, 130.0, 164.0, 131.0, 136.0, 129.0, 150.0, 146.0, 133.0, 96.0, 139.0, 119.0, 108.0, 124.0, 109.0, 114.0, 113.0, 123.0, 125.0, 129.0, 99.0, 159.0, 109.0, 115.0, 127.0, 128.0, 101.0, 122.0, 118.0, 113.0, 110.0, 107.0, 112.0, 89.0, 107.0, 118.0, 89.0, 101.0, 127.0, 125.0, 111.0, 110.0, 121.0, 125.0, 111.0, 123.0, 109.0, 116.0, 118.0, 107.0, 87.0, 105.0, 121.0, 111.0, 127.0, 128.0, 116.0, 128.0, 116.0, 112.0, 135.0, 122.0, 106.0, 97.0, 100.0, 121.0, 94.0, 117.0, 124.0, 93.0, 116.0, 99.0, 114.0, 107.0, 96.0, 105.0, 102.0, 84.0, 138.0, 100.0, 100.0, 115.0, 133.0, 101.0, 99.0, 105.0, 116.0, 109.0, 100.0, 109.0, 120.0, 131.0, 107.0, 110.0, 111.0, 98.0, 118.0, 97.0, 122.0, 115.0, 121.0, 114.0, 91.0, 86.0, 116.0, 85.0, 79.0, 99.0, 97.0, 89.0, 103.0, 78.0, 108.0, 107.0, 78.0, 101.0, 99.0, 96.0, 119.0, 87.0, 98.0, 113.0, 112.0, 101.0, 78.0, 125.0, 101.0, 102.0, 137.0, 85.0, 97.0, 96.0, 119.0, 119.0, 93.0, 84.0, 94.0, 91.0, 132.0, 108.0, 113.0, 98.0, 127.0, 102.0, 88.0, 93.0, 124.0, 102.0, 99.0, 97.0, 99.0, 85.0, 103.0, 94.0, 108.0, 116.0, 103.0, 114.0, 105.0, 123.0, 122.0, 94.0, 104.0, 101.0, 103.0, 109.0, 115.0, 117.0, 125.0, 81.0, 115.0, 112.0, 116.0, 100.0, 108.0, 105.0, 97.0, 101.0, 105.0, 98.0, 124.0, 98.0, 101.0, 103.0, 123.0, 124.0, 122.0, 115.0, 102.0, 115.0, 116.0, 122.0, 111.0, 88.0, 99.0, 95.0, 112.0, 122.0, 131.0, 110.0, 112.0, 96.0, 108.0, 100.0, 103.0, 106.0, 119.0, 104.0, 102.0, 97.0, 125.0, 93.0, 117.0, 133.0, 112.0, 137.0, 110.0, 104.0, 120.0, 115.0, 111.0, 118.0, 113.0, 100.0, 125.0, 108.0, 109.0, 122.0, 99.0, 128.0, 105.0, 112.0, 122.0, 112.0, 114.0, 109.0, 108.0, 111.0, 113.0, 114.0, 105.0, 101.0, 110.0, 104.0, 112.0, 114.0, 109.0, 92.0, 111.0, 102.0, 91.0, 119.0, 111.0, 95.0, 107.0, 97.0, 115.0, 108.0, 124.0, 118.0, 123.0, 119.0, 122.0, 112.0, 106.0, 101.0, 93.0, 116.0, 123.0, 112.0, 120.0, 87.0, 102.0, 116.0, 113.0, 118.0, 135.0, 110.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [65.0, 71.0, 74.0, 78.0, 68.0, 65.0, 79.0, 104.0, 95.0, 118.0, 116.0, 161.0, 141.0, 148.0, 182.0, 146.0, 164.0, 199.0, 174.0, 205.0, 166.0, 167.0, 186.0, 158.0, 195.0, 179.0, 188.0, 208.0, 187.0, 145.0, 145.0, 146.0, 156.0, 175.0, 132.0, 180.0, 177.0, 205.0, 172.0, 159.0, 158.0, 175.0, 153.0, 203.0, 196.0, 170.0, 185.0, 179.0, 140.0, 227.0, 198.0, 165.0, 172.0, 149.0, 199.0, 213.0, 179.0, 157.0, 255.0, 240.0, 186.0, 191.0, 164.0, 186.0, 208.0, 229.0, 213.0, 198.0, 198.0, 178.0, 246.0, 222.0, 177.0, 236.0, 193.0, 215.0, 226.0, 205.0, 251.0, 226.0, 224.0, 245.0, 219.0, 205.0, 198.0, 190.0, 171.0, 191.0, 171.0, 187.0, 182.0, 207.0, 233.0, 201.0, 220.0, 152.0, 216.0, 194.0, 175.0, 157.0, 165.0, 188.0, 163.0, 163.0, 160.0, 155.0, 160.0, 167.0, 144.0, 190.0, 194.0, 143.0, 153.0, 175.0, 158.0, 147.0, 166.0, 115.0, 142.0, 141.0, 117.0, 131.0, 132.0, 130.0, 164.0, 131.0, 136.0, 129.0, 150.0, 146.0, 133.0, 96.0, 139.0, 119.0, 108.0, 124.0, 109.0, 114.0, 113.0, 123.0, 125.0, 129.0, 99.0, 159.0, 109.0, 115.0, 127.0, 128.0, 101.0, 122.0, 118.0, 113.0, 110.0, 107.0, 112.0, 89.0, 107.0, 118.0, 89.0, 101.0, 127.0, 125.0, 111.0, 110.0, 121.0, 125.0, 111.0, 123.0, 109.0, 116.0, 118.0, 107.0, 87.0, 105.0, 121.0, 111.0, 127.0, 128.0, 116.0, 128.0, 116.0, 112.0, 135.0, 122.0, 106.0, 97.0, 100.0, 121.0, 94.0, 117.0, 124.0, 93.0, 116.0, 99.0, 114.0, 107.0, 96.0, 105.0, 102.0, 84.0, 138.0, 100.0, 100.0, 115.0, 133.0, 101.0, 99.0, 105.0, 116.0, 109.0, 100.0, 109.0, 120.0, 131.0, 107.0, 110.0, 111.0, 98.0, 118.0, 97.0, 122.0, 115.0, 121.0, 114.0, 91.0, 86.0, 116.0, 85.0, 79.0, 99.0, 97.0, 89.0, 103.0, 78.0, 108.0, 107.0, 78.0, 101.0, 99.0, 96.0, 119.0, 87.0, 98.0, 113.0, 112.0, 101.0, 78.0, 125.0, 101.0, 102.0, 137.0, 85.0, 97.0, 96.0, 119.0, 119.0, 93.0, 84.0, 94.0, 91.0, 132.0, 108.0, 113.0, 98.0, 127.0, 102.0, 88.0, 93.0, 124.0, 102.0, 99.0, 97.0, 99.0, 85.0, 103.0, 94.0, 108.0, 116.0, 103.0, 114.0, 105.0, 123.0, 122.0, 94.0, 104.0, 101.0, 103.0, 109.0, 115.0, 117.0, 125.0, 81.0, 115.0, 112.0, 116.0, 100.0, 108.0, 105.0, 97.0, 101.0, 105.0, 98.0, 124.0, 98.0, 101.0, 103.0, 123.0, 124.0, 122.0, 115.0, 102.0, 115.0, 116.0, 122.0, 111.0, 88.0, 99.0, 95.0, 112.0, 122.0, 131.0, 110.0, 112.0, 96.0, 108.0, 100.0, 103.0, 106.0, 119.0, 104.0, 102.0, 97.0, 125.0, 93.0, 117.0, 133.0, 112.0, 137.0, 110.0, 104.0, 120.0, 115.0, 111.0, 118.0, 113.0, 100.0, 125.0, 108.0, 109.0, 122.0, 99.0, 128.0, 105.0, 112.0, 122.0, 112.0, 114.0, 109.0, 108.0, 111.0, 113.0, 114.0, 105.0, 101.0, 110.0, 104.0, 112.0, 114.0, 109.0, 92.0, 111.0, 102.0, 91.0, 119.0, 111.0, 95.0, 107.0, 97.0, 115.0, 108.0, 124.0, 118.0, 123.0, 119.0, 122.0, 112.0, 106.0, 101.0, 93.0, 116.0, 123.0, 112.0, 120.0, 87.0, 102.0, 116.0, 113.0, 118.0, 135.0, 110.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.02467, 180.43758, 180.43758, 180.43758, 180.43777, 180.43793, 180.43831, 180.43793, 180.43802, 180.43829, 180.42958, 180.42429, 180.42331, 180.41566, 180.40419, 180.39563, 180.38898, 180.38272, 180.37767, 180.36685, 180.3624, 180.35443, 180.34731, 180.34276, 180.34059, 180.33824, 180.33578, 180.33348, 180.33122, 180.32733, 180.32587, 180.33919, 180.33737, 180.34363, 180.342, 180.33069, 180.32681, 180.32585, 180.32092, 180.32108, 180.32138, 180.32286, 180.32475, 180.31996, 180.32315, 180.32661, 180.32353, 180.32631, 180.3324, 180.33495, 180.34297, 180.3517, 180.3622, 180.37233, 180.38312, 180.39899, 180.40675, 180.41966, 180.43352, 180.4433, 180.45685, 180.47159, 180.48726, 180.50627, 180.52652, 180.54735, 180.56612, 180.58878, 180.60876, 180.63284, 180.65726, 180.68228, 180.70798, 180.73802, 180.77544, 180.79449, 180.82513, 180.85388, 180.88318, 180.90952, 180.93996, 180.9704, 181.00171, 181.03206, 181.06531, 181.1013, 181.13477, 181.15898, 181.19191, 181.22948, 181.26605, 181.30205, 181.33176, 181.36722, 181.40222, 181.43898, 181.4686, 181.50232, 181.53323, 181.56693, 181.60017, 181.63365, 181.66275, 181.69737, 181.73155, 181.76347, 181.8042, 181.83623, 181.86909, 181.90247, 181.93695, 181.96951, 182.00578, 182.04301, 182.07603, 182.11412, 182.15521, 182.18857, 182.22928, 182.26672, 182.3042, 182.34148, 182.37926, 182.41901, 182.45923, 182.49518, 182.53793, 182.57965, 182.61847, 182.65536, 182.6929, 182.72876, 182.76958, 182.80853, 182.85202, 182.88937, 182.92555, 182.96187, 182.99063, 183.02582, 183.05833, 183.08974, 183.12651, 183.16095, 183.19424, 183.233, 183.26149, 183.29265, 183.32909, 183.36882, 183.40269, 183.43456, 183.47014, 183.51022, 183.54683, 183.57953, 183.61252, 183.64738, 183.68155, 183.71558, 183.75716, 183.79567, 183.83615, 183.87654, 183.9173, 183.9584, 184.00073, 184.04141, 184.08711, 184.12192, 184.16089, 184.19904, 184.23912, 184.27597, 184.31317, 184.35162, 184.39233, 184.43021, 184.46562, 184.50061, 184.54076, 184.5798, 184.62137, 184.66426, 184.70601, 184.74544, 184.7812, 184.8163, 184.85382, 184.89362, 184.9332, 184.9715, 185.00937, 185.05093, 185.09132, 185.12502, 185.16487, 185.20316, 185.24188, 185.27464, 185.31422, 185.35551, 185.3972, 185.43919, 185.47906, 185.52074, 185.56161, 185.60054, 185.64554, 185.68713, 185.72649, 185.76546, 185.80576, 185.84767, 185.89198, 185.9361, 185.98022, 186.01895, 186.05711, 186.10294, 186.13905, 186.17926, 186.22005, 186.25861, 186.29631, 186.33633, 186.37819, 186.41498, 186.452, 186.48996, 186.52638, 186.56227, 186.59106, 186.62415, 186.66559, 186.70592, 186.74504, 186.78651, 186.83006, 186.87518, 186.91788, 186.96049, 187.00543, 187.05008, 187.09511, 187.13741, 187.17758, 187.21588, 187.25984, 187.30086, 187.34575, 187.39095, 187.43542, 187.4792, 187.51852, 187.56268, 187.60396, 187.64711, 187.68872, 187.73135, 187.77692, 187.81973, 187.86543, 187.91296, 187.96025, 188.00529, 188.04802, 188.0909, 188.13518, 188.18434, 188.22716, 188.27409, 188.32169, 188.36803, 188.41319, 188.45816, 188.50641, 188.54868, 188.59381, 188.6367, 188.68343, 188.72693, 188.77374, 188.8172, 188.86154, 188.90767, 188.95059, 188.99326, 189.04083, 189.08832, 189.13934, 189.1855, 189.2296, 189.27489, 189.32558, 189.36694, 189.41133, 189.45744, 189.50322, 189.54796, 189.59531, 189.6389, 189.68634, 189.73462, 189.78769, 189.83501, 189.88196, 189.92941, 189.97726, 190.02953, 190.08095, 190.13335, 190.18449, 190.23326, 190.28383, 190.33415, 190.38512, 190.43832, 190.49026, 190.5453, 190.59666, 190.65088, 190.70216, 190.75441, 190.80804, 190.85649, 190.90819, 190.957, 191.00778, 191.05713, 191.10803, 191.15628, 191.20445, 191.25539, 191.30585, 191.35631, 191.40929, 191.46144, 191.5153, 191.5732, 191.6273, 191.67821, 191.73494, 191.79005, 191.84462, 191.89845, 191.95538, 192.01093, 192.06554, 192.1189, 192.17081, 192.2244, 192.2774, 192.33224, 192.38445, 192.44177, 192.49707, 192.55254, 192.60464, 192.65576, 192.70808, 192.76437, 192.82317, 192.88344, 192.93953, 192.99843, 193.05219, 193.1062, 193.16641, 193.22375, 193.28175, 193.3349, 193.39145, 193.44878, 193.50717, 193.55751, 193.61333, 193.66898, 193.72675, 193.79041, 193.84534, 193.90236, 193.96567, 194.0249, 194.08501, 194.14468, 194.2052, 194.2684, 194.32666, 194.38776, 194.44768, 194.50999, 194.57324, 194.63622, 194.69333, 194.74876, 194.80455, 194.86299, 194.92128, 194.97459]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.02467, 180.43758, 180.43758, 180.43758, 180.43777, 180.43793, 180.43831, 180.43793, 180.43802, 180.43829, 180.42958, 180.42429, 180.42331, 180.41566, 180.40419, 180.39563, 180.38898, 180.38272, 180.37767, 180.36685, 180.3624, 180.35443, 180.34731, 180.34276, 180.34059, 180.33824, 180.33578, 180.33348, 180.33122, 180.32733, 180.32587, 180.33919, 180.33737, 180.34363, 180.342, 180.33069, 180.32681, 180.32585, 180.32092, 180.32108, 180.32138, 180.32286, 180.32475, 180.31996, 180.32315, 180.32661, 180.32353, 180.32631, 180.3324, 180.33495, 180.34297, 180.3517, 180.3622, 180.37233, 180.38312, 180.39899, 180.40675, 180.41966, 180.43352, 180.4433, 180.45685, 180.47159, 180.48726, 180.50627, 180.52652, 180.54735, 180.56612, 180.58878, 180.60876, 180.63284, 180.65726, 180.68228, 180.70798, 180.73802, 180.77544, 180.79449, 180.82513, 180.85388, 180.88318, 180.90952, 180.93996, 180.9704, 181.00171, 181.03206, 181.06531, 181.1013, 181.13477, 181.15898, 181.19191, 181.22948, 181.26605, 181.30205, 181.33176, 181.36722, 181.40222, 181.43898, 181.4686, 181.50232, 181.53323, 181.56693, 181.60017, 181.63365, 181.66275, 181.69737, 181.73155, 181.76347, 181.8042, 181.83623, 181.86909, 181.90247, 181.93695, 181.96951, 182.00578, 182.04301, 182.07603, 182.11412, 182.15521, 182.18857, 182.22928, 182.26672, 182.3042, 182.34148, 182.37926, 182.41901, 182.45923, 182.49518, 182.53793, 182.57965, 182.61847, 182.65536, 182.6929, 182.72876, 182.76958, 182.80853, 182.85202, 182.88937, 182.92555, 182.96187, 182.99063, 183.02582, 183.05833, 183.08974, 183.12651, 183.16095, 183.19424, 183.233, 183.26149, 183.29265, 183.32909, 183.36882, 183.40269, 183.43456, 183.47014, 183.51022, 183.54683, 183.57953, 183.61252, 183.64738, 183.68155, 183.71558, 183.75716, 183.79567, 183.83615, 183.87654, 183.9173, 183.9584, 184.00073, 184.04141, 184.08711, 184.12192, 184.16089, 184.19904, 184.23912, 184.27597, 184.31317, 184.35162, 184.39233, 184.43021, 184.46562, 184.50061, 184.54076, 184.5798, 184.62137, 184.66426, 184.70601, 184.74544, 184.7812, 184.8163, 184.85382, 184.89362, 184.9332, 184.9715, 185.00937, 185.05093, 185.09132, 185.12502, 185.16487, 185.20316, 185.24188, 185.27464, 185.31422, 185.35551, 185.3972, 185.43919, 185.47906, 185.52074, 185.56161, 185.60054, 185.64554, 185.68713, 185.72649, 185.76546, 185.80576, 185.84767, 185.89198, 185.9361, 185.98022, 186.01895, 186.05711, 186.10294, 186.13905, 186.17926, 186.22005, 186.25861, 186.29631, 186.33633, 186.37819, 186.41498, 186.452, 186.48996, 186.52638, 186.56227, 186.59106, 186.62415, 186.66559, 186.70592, 186.74504, 186.78651, 186.83006, 186.87518, 186.91788, 186.96049, 187.00543, 187.05008, 187.09511, 187.13741, 187.17758, 187.21588, 187.25984, 187.30086, 187.34575, 187.39095, 187.43542, 187.4792, 187.51852, 187.56268, 187.60396, 187.64711, 187.68872, 187.73135, 187.77692, 187.81973, 187.86543, 187.91296, 187.96025, 188.00529, 188.04802, 188.0909, 188.13518, 188.18434, 188.22716, 188.27409, 188.32169, 188.36803, 188.41319, 188.45816, 188.50641, 188.54868, 188.59381, 188.6367, 188.68343, 188.72693, 188.77374, 188.8172, 188.86154, 188.90767, 188.95059, 188.99326, 189.04083, 189.08832, 189.13934, 189.1855, 189.2296, 189.27489, 189.32558, 189.36694, 189.41133, 189.45744, 189.50322, 189.54796, 189.59531, 189.6389, 189.68634, 189.73462, 189.78769, 189.83501, 189.88196, 189.92941, 189.97726, 190.02953, 190.08095, 190.13335, 190.18449, 190.23326, 190.28383, 190.33415, 190.38512, 190.43832, 190.49026, 190.5453, 190.59666, 190.65088, 190.70216, 190.75441, 190.80804, 190.85649, 190.90819, 190.957, 191.00778, 191.05713, 191.10803, 191.15628, 191.20445, 191.25539, 191.30585, 191.35631, 191.40929, 191.46144, 191.5153, 191.5732, 191.6273, 191.67821, 191.73494, 191.79005, 191.84462, 191.89845, 191.95538, 192.01093, 192.06554, 192.1189, 192.17081, 192.2244, 192.2774, 192.33224, 192.38445, 192.44177, 192.49707, 192.55254, 192.60464, 192.65576, 192.70808, 192.76437, 192.82317, 192.88344, 192.93953, 192.99843, 193.05219, 193.1062, 193.16641, 193.22375, 193.28175, 193.3349, 193.39145, 193.44878, 193.50717, 193.55751, 193.61333, 193.66898, 193.72675, 193.79041, 193.84534, 193.90236, 193.96567, 194.0249, 194.08501, 194.14468, 194.2052, 194.2684, 194.32666, 194.38776, 194.44768, 194.50999, 194.57324, 194.63622, 194.69333, 194.74876, 194.80455, 194.86299, 194.92128, 194.97459]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [26.15537, 1.59225, 1.58677, 1.61174, 1.60131, 1.58979, 1.6009, 1.60255, 1.59989, 1.59397, 1.59991, 1.60879, 1.59752, 1.58326, 1.60593, 1.58196, 1.58281, 1.58285, 1.65512, 1.58951, 1.57778, 1.59099, 1.59905, 1.5964, 1.60421, 1.59987, 1.60383, 1.59456, 1.59474, 1.60292, 1.59587, 1.59615, 1.59953, 1.68491, 1.61405, 1.61646, 1.76204, 1.6157, 1.60582, 1.60949, 1.60517, 1.60169, 1.5944, 1.59771, 1.59812, 1.61186, 1.60798, 1.59786, 1.69134, 1.607, 1.62116, 1.61495, 1.61958, 1.61282, 1.60615, 1.61947, 1.6053, 1.59812, 1.60103, 1.61637, 1.60915, 1.61703, 1.61268, 1.61077, 1.61236, 1.61876, 1.60773, 1.69396, 1.60939, 1.61301, 1.62827, 1.61429, 1.61159, 1.60859, 1.61405, 1.62895, 1.61614, 1.61446, 1.60675, 1.61067, 1.61896, 1.61461, 1.61244, 1.60436, 1.6079, 1.619, 1.61303, 1.61117, 1.61223, 1.60766, 1.62186, 1.60682, 1.60832, 1.60625, 1.60469, 1.61342, 1.60768, 1.60669, 1.59722, 1.69938, 1.61072, 1.61909, 1.61007, 1.6046, 1.60277, 1.61264, 1.61634, 1.61492, 1.61043, 1.62152, 1.61505, 1.61393, 1.61336, 1.61268, 1.61629, 1.61635, 1.62076, 1.61243, 1.61515, 1.61244, 1.61769, 1.61729, 1.60493, 1.60897, 1.61012, 1.61259, 1.6206, 1.60935, 1.61072, 1.61412, 1.62132, 1.61512, 1.61556, 1.61045, 1.6109, 1.61406, 1.61499, 1.60648, 1.62368, 1.61793, 1.62077, 1.61115, 1.607, 1.60097, 1.60715, 1.61148, 1.61713, 1.61144, 1.62249, 1.61481, 1.61115, 1.6037, 1.61119, 1.60767, 1.6172, 1.61279, 1.60574, 1.60707, 1.60482, 1.60401, 1.61113, 1.61346, 1.60704, 1.61142, 1.60677, 1.60612, 1.59885, 1.60751, 1.60394, 1.60565, 1.60074, 1.60646, 1.60139, 1.60114, 1.60502, 1.59931, 1.59106, 1.59528, 1.59562, 1.60655, 1.61019, 1.60604, 1.60255, 1.59481, 1.59218, 1.59628, 1.58975, 1.60275, 1.59914, 1.59723, 1.59728, 1.58386, 1.61425, 1.60353, 1.60061, 1.60375, 1.61192, 1.61512, 1.60494, 1.59982, 1.59392, 1.59773, 1.59899, 1.60034, 1.59034, 1.59986, 1.59404, 1.59171, 1.58924, 1.58292, 1.59951, 1.58972, 1.60076, 1.59525, 1.60354, 1.60474, 1.6007, 1.60461, 1.60303, 1.68738, 1.61462, 1.6112, 1.60314, 1.60468, 1.60954, 1.61515, 1.60446, 1.60607, 1.60574, 1.60376, 1.60767, 1.60168, 1.60809, 1.60685, 1.59979, 1.59981, 1.59996, 1.60233, 1.61191, 1.60192, 1.60578, 1.61979, 1.6159, 1.61226, 1.6128, 1.60991, 1.62187, 1.61382, 1.60853, 1.61365, 1.6207, 1.63823, 1.61317, 1.60999, 1.6096, 1.6053, 1.62098, 1.60515, 1.61012, 1.60877, 1.61097, 1.62766, 1.61189, 1.61276, 1.61683, 1.61267, 1.62231, 1.61022, 1.61488, 1.61227, 1.60799, 1.61989, 1.61118, 1.60947, 1.61635, 1.60971, 1.61707, 1.61308, 1.60535, 1.61359, 1.60892, 1.61075, 1.60793, 1.60987, 1.61295, 1.61056, 1.60924, 1.61593, 1.60828, 1.62137, 1.60777, 1.6163, 1.61976, 1.60496, 1.61232, 1.60943, 1.60387, 1.61497, 1.60986, 1.61254, 1.61053, 1.61641, 1.62112, 1.60996, 1.62043, 1.61238, 1.61482, 1.61865, 1.61289, 1.61175, 1.61784, 1.61203, 1.6132, 1.60843, 1.61847, 1.61033, 1.6185, 1.61766, 1.6264, 1.62151, 1.62048, 1.61539, 1.61807, 1.61346, 1.60979, 1.61291, 1.61433, 1.61137, 1.616, 1.60714, 1.6154, 1.61351, 1.60767, 1.60384, 1.60001, 1.59921, 1.60103, 1.60417, 1.60117, 1.59284, 1.60079, 1.59673, 1.59125, 1.59593, 1.59394, 1.59478, 1.59263, 1.59408, 1.59955, 1.66468, 1.59302, 1.59156, 1.59525, 1.62673, 1.61448, 1.60772, 1.60098, 1.6066, 1.62998, 1.62933, 1.6147, 1.61299, 1.61044, 1.62556, 1.61734, 1.61197, 1.61149, 1.61287, 1.62523, 1.61258, 1.60355, 1.6117, 1.61092, 1.60763, 1.61177, 1.61161, 1.6207, 1.61553, 1.62712, 1.62883, 1.6176, 1.62185, 1.60923, 1.61676, 1.62142, 1.62074, 1.61866, 1.61459, 1.59668, 1.61134, 1.60642, 1.60975, 1.61506, 1.60601, 1.62434, 1.61024, 1.61231, 1.61973, 1.61419, 1.61888]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.5974]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.5974]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [269.72311]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [269.72311]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/model_config.yaml
new file mode 100644
index 0000000000..de27041eba
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_native_fp8_tp_pp_sp/model_config.yaml
@@ -0,0 +1,55 @@
+ENV_VARS:
+  CUDA_DEVICE_MAX_CONNECTIONS: 1
+  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
+  NCCL_ALGO: Tree
+  CUBLAS_WORKSPACE_CONFIG: :4096:8
+MODEL_ARGS:
+  --num-layers: 12
+  --hidden-size: 512
+  --num-attention-heads: 8
+  --log-params-norm: true
+  --log-num-zeros-in-grad: true
+  --log-validation-ppl-to-tensorboard: true
+  --log-timers-to-tensorboard: true
+  --tensorboard-dir: ${TENSORBOARD_PATH}
+  --micro-batch-size: 2
+  --global-batch-size: 128
+  --seq-length: 1024
+  --max-position-embeddings: 1024
+  --train-iters: 2000
+  --timing-log-level: 2
+  --lr-decay-iters: 320000
+  --save: ${CHECKPOINT_PATH}
+  --load: ${CHECKPOINT_PATH}
+  --data-path: ${DATA_PATH}/my-gpt3_00_text_document
+  --vocab-file: ${DATA_PATH}/bpe/vocab.json
+  --merge-file: ${DATA_PATH}/bpe/merges.txt
+  --split: 949,50,1
+  --distributed-backend: nccl
+  --lr: 0.00015
+  --lr-decay-style: cosine
+  --min-lr: 1.0e-5
+  --weight-decay: 1e-2
+  --clip-grad: 1.0
+  --lr-warmup-fraction: .01
+  --log-interval: 1
+  --save-interval: 10000
+  --eval-interval: 1000
+  --eval-iters: 10
+  --transformer-impl: transformer_engine
+  --tensor-model-parallel-size: 2
+  --pipeline-model-parallel-size: 2
+  --sequence-parallel: true
+  --deterministic-mode: true
+  --no-gradient-accumulation-fusion: true
+  --fp8-format: hybrid
+  --fp8-amax-history-len: 1024
+  --fp8-amax-compute-algo: max
+  --fp8-param-gather: true
+  --use-distributed-optimizer: true
+  --attention-softmax-in-fp32: true
+  --ckpt-format: torch_dist
+  --data-cache-path: ${DATA_CACHE_PATH}
+  --bf16: true
+TEST_TYPE: regular
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/golden_values.json b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/golden_values.json
new file mode 100644
index 0000000000..3d10208bdb
--- /dev/null
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/golden_values.json
@@ -0,0 +1 @@
+{"forward-backward-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [23.87084, 2.7908, 2.78539, 2.7894, 2.7852, 2.79146, 2.78472, 2.78272, 2.79513, 2.79226, 2.78492, 2.79008, 2.7883, 2.79109, 2.79145, 2.79405, 2.79452, 2.79382, 2.79611, 2.79622, 2.79284, 2.79072, 2.79713, 2.79936, 2.79764, 2.78902, 2.79179, 2.79398, 2.79758, 2.78776, 2.79263, 2.79691, 2.80152, 2.80908, 2.80472, 2.79568, 2.80506, 2.80202, 2.80799, 2.80521, 2.80461, 2.8094, 2.80343, 2.80761, 2.81112, 2.81918, 2.80453, 2.80312, 2.80829, 2.80344, 2.80562, 2.80427, 2.79734, 2.81406, 2.90515, 2.82407, 2.81478, 2.81303, 2.81592, 2.81601, 2.82191, 2.81825, 2.82313, 2.81813, 2.8193, 2.81849, 2.80988, 2.81403, 2.81327, 2.80905, 2.80847, 2.80536, 2.80854, 2.8101, 2.81145, 2.80684, 2.81147, 2.81242, 2.80609, 2.80189, 2.79515, 2.7996, 2.80311, 2.8045, 2.80721, 2.80272, 2.81517, 2.80665, 2.81404, 2.81132, 2.80918, 2.80977, 2.80802, 2.80672, 2.80661, 2.80353, 2.81098, 2.80324, 2.80589, 2.80502, 2.80911, 2.80853, 2.80753, 2.80189, 2.80083, 2.8104, 2.80739, 2.80143, 2.8113, 2.80321, 2.80139, 2.79801, 2.80488, 2.80348, 2.80222, 2.80147, 2.80475, 2.79774, 2.79626, 2.80141, 2.80405, 2.80603, 2.80138, 2.80245, 2.79478, 2.80184, 2.80852, 2.8046, 2.81228, 2.80607, 2.80189, 2.80761, 2.80561, 2.8108, 2.79699, 2.80217, 2.82211, 2.79924, 2.81403, 2.80853, 2.8231, 2.81577, 2.8231, 2.82156, 2.81887, 2.82238, 2.81839, 2.82501, 2.81996, 2.82429, 2.82644, 2.82806, 2.82682, 2.8177, 2.81557, 2.82321, 2.80343, 2.83308, 2.81556, 2.80394, 2.8065, 2.80837, 2.80217, 2.81017, 2.80941, 2.80836, 2.80137, 2.80618, 2.8106, 2.81859, 2.81372, 2.80415, 2.81048, 2.80289, 2.8074, 2.80851, 2.80327, 2.80386, 2.80501, 2.80423, 2.80829, 2.80479, 2.80551, 2.80503, 2.80867, 2.80686, 2.80919, 2.80825, 2.80825, 2.80524, 2.8104, 2.81017, 2.8092, 2.80887, 2.80127, 2.80865, 2.81409, 2.81338, 2.81622, 2.81551, 2.78402, 2.78667, 2.77607, 2.78149, 2.79485, 2.77794, 2.77679, 2.77522, 2.77183, 2.76873, 2.76746, 2.78341, 2.77337, 2.77333, 2.77216, 2.76418, 2.77521, 2.77572, 2.77007, 2.77107, 2.77433, 2.7767, 2.77171, 2.78519, 2.77337, 2.77435, 2.77481, 2.77069, 2.77522, 2.77587, 2.78393, 2.7743, 2.78225, 2.77729, 2.7811, 2.77531, 2.77781, 2.77542, 2.76967, 2.77202, 2.77351, 2.78458, 2.77568, 2.78594, 2.7783, 2.78007, 2.78444, 2.77342, 2.77788, 2.8174, 2.80994, 2.81175, 2.8116, 2.80961, 2.81294, 2.80664, 2.82069, 2.80473, 2.80257, 2.80502, 2.79658, 2.80824, 2.80374, 2.80925, 2.80871, 2.80288, 2.82051, 2.81324, 2.81301, 2.81015, 2.81433, 2.81771, 2.82163, 2.82047, 2.84243, 2.82391, 2.82193, 2.82874, 2.82499, 2.82329, 2.82269, 2.78491, 2.78347, 2.78283, 2.77915, 2.78184, 2.78745, 2.77885, 2.78616, 2.78454, 2.79387, 2.78599, 2.78264, 2.78415, 2.77954, 2.78012, 2.77574, 2.77417, 2.77157, 2.77598, 2.78523, 2.78094, 2.77956, 2.78155, 2.76974, 2.76609, 2.77059, 2.7715, 2.77799, 2.78545, 2.79125, 2.78957, 2.7735, 2.77351, 2.77438, 2.77082, 2.76702, 2.76913, 2.77001, 2.77136, 2.77805, 2.77172, 2.77423, 2.77469, 2.76739, 2.76274, 2.76413, 2.769, 2.7747, 2.77447, 2.77236, 2.77322, 2.77126, 2.76432, 2.77139, 2.75782, 2.76437, 2.77311, 2.77485, 2.77226, 2.7716, 2.77527, 2.76108, 2.76967, 2.76835, 2.76738, 2.77531, 2.77528, 2.76726, 2.77204, 2.76615, 2.76217, 2.76346, 2.76358, 2.86867, 2.76052, 2.76931, 2.77037, 2.76368, 2.76923, 2.76194, 2.77432, 2.77035, 2.76442, 2.77453, 2.76955, 2.75944, 2.76101, 2.76318, 2.76891, 2.7675, 2.77756, 2.77522, 2.76826, 2.76436, 2.77785, 2.77783, 2.76832, 2.76347, 2.76291, 2.77118, 2.76677, 2.76612, 2.76582, 2.76273, 2.75857, 2.75873, 2.7722, 2.76177, 2.77171, 2.77644, 2.7639, 2.7721, 2.76437, 2.76496, 2.78781, 2.7708, 2.77914, 2.7677, 2.77621]}, "forward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [9.51205, 1.43678, 1.43791, 1.4403, 1.43427, 1.43756, 1.43758, 1.43562, 1.44189, 1.44431, 1.43685, 1.43669, 1.43665, 1.43656, 1.44116, 1.44015, 1.44001, 1.44016, 1.4435, 1.44113, 1.44161, 1.44108, 1.44253, 1.44731, 1.44571, 1.43765, 1.44091, 1.44413, 1.44785, 1.43882, 1.44323, 1.43963, 1.44096, 1.44584, 1.4433, 1.43872, 1.44424, 1.44585, 1.4456, 1.44851, 1.44579, 1.4472, 1.44488, 1.44427, 1.44702, 1.44843, 1.44696, 1.44174, 1.44868, 1.44573, 1.44263, 1.44873, 1.44368, 1.45098, 1.50386, 1.46222, 1.45889, 1.46823, 1.45958, 1.46199, 1.45939, 1.46248, 1.46055, 1.46617, 1.46663, 1.46838, 1.45647, 1.45342, 1.45158, 1.44745, 1.45071, 1.44757, 1.45057, 1.45354, 1.45015, 1.45365, 1.45031, 1.45396, 1.44855, 1.44723, 1.44555, 1.44612, 1.44775, 1.44969, 1.45014, 1.4487, 1.447, 1.44896, 1.4498, 1.45306, 1.45037, 1.4495, 1.44838, 1.44482, 1.45215, 1.448, 1.45159, 1.44448, 1.44896, 1.44752, 1.44756, 1.45023, 1.45026, 1.44675, 1.44444, 1.45064, 1.44643, 1.44631, 1.45024, 1.44933, 1.44526, 1.44522, 1.44467, 1.4481, 1.44864, 1.45043, 1.45185, 1.44907, 1.44793, 1.45106, 1.44909, 1.44946, 1.44262, 1.43975, 1.44103, 1.44743, 1.45025, 1.4482, 1.45283, 1.44737, 1.44579, 1.44509, 1.44631, 1.44428, 1.44535, 1.45213, 1.45201, 1.44741, 1.45012, 1.45313, 1.47204, 1.46712, 1.47171, 1.47404, 1.47244, 1.46786, 1.46879, 1.46914, 1.47064, 1.46718, 1.47001, 1.47261, 1.47278, 1.46528, 1.46833, 1.46966, 1.44696, 1.45977, 1.44861, 1.44782, 1.44378, 1.44407, 1.44816, 1.45245, 1.449, 1.44784, 1.4449, 1.44523, 1.44905, 1.45312, 1.44739, 1.44742, 1.45369, 1.44478, 1.44662, 1.44949, 1.4459, 1.4448, 1.44385, 1.44392, 1.45267, 1.44333, 1.44892, 1.44724, 1.4485, 1.44583, 1.44996, 1.4476, 1.4446, 1.44975, 1.451, 1.45004, 1.44925, 1.45149, 1.44617, 1.44967, 1.44957, 1.45131, 1.45283, 1.4513, 1.42552, 1.41683, 1.41289, 1.41323, 1.41749, 1.41143, 1.41101, 1.4112, 1.4135, 1.41006, 1.4137, 1.41016, 1.41535, 1.41173, 1.41324, 1.40716, 1.40976, 1.40928, 1.41, 1.40851, 1.40949, 1.41481, 1.40726, 1.41247, 1.40893, 1.40726, 1.41201, 1.41338, 1.41944, 1.41452, 1.41165, 1.41022, 1.41318, 1.41802, 1.41449, 1.41063, 1.41492, 1.41265, 1.41132, 1.41365, 1.41475, 1.41847, 1.41122, 1.41128, 1.41301, 1.41405, 1.41415, 1.41581, 1.41619, 1.42827, 1.42088, 1.42041, 1.42456, 1.42192, 1.42307, 1.42073, 1.42805, 1.42078, 1.42396, 1.42359, 1.42048, 1.42105, 1.41976, 1.4247, 1.42503, 1.42186, 1.42845, 1.42785, 1.42791, 1.4201, 1.42849, 1.42307, 1.43185, 1.43491, 1.44341, 1.43591, 1.44767, 1.44319, 1.43803, 1.4396, 1.43766, 1.41441, 1.41492, 1.41502, 1.41802, 1.41644, 1.41395, 1.4088, 1.41436, 1.41116, 1.41904, 1.41497, 1.4117, 1.41375, 1.41211, 1.41098, 1.41349, 1.40846, 1.41118, 1.41363, 1.41608, 1.41063, 1.40863, 1.40931, 1.40576, 1.40253, 1.40633, 1.4031, 1.40517, 1.40582, 1.40973, 1.41428, 1.41255, 1.41129, 1.4127, 1.41154, 1.40611, 1.40611, 1.40794, 1.41156, 1.40745, 1.41035, 1.4097, 1.40988, 1.40878, 1.40716, 1.40765, 1.41137, 1.4109, 1.40902, 1.41507, 1.40796, 1.41525, 1.40249, 1.40831, 1.39916, 1.40546, 1.40999, 1.41032, 1.41283, 1.41312, 1.40738, 1.40936, 1.40757, 1.41053, 1.40694, 1.40948, 1.41066, 1.40854, 1.40655, 1.41367, 1.41378, 1.40999, 1.41174, 1.51942, 1.40444, 1.4119, 1.41683, 1.40936, 1.41487, 1.40883, 1.41143, 1.41268, 1.40887, 1.41527, 1.41408, 1.41281, 1.41183, 1.4134, 1.4109, 1.41349, 1.41109, 1.41503, 1.4111, 1.40948, 1.41361, 1.41212, 1.40741, 1.40997, 1.41405, 1.41032, 1.40943, 1.40908, 1.40969, 1.40965, 1.40759, 1.41424, 1.41408, 1.41111, 1.41223, 1.4114, 1.41026, 1.41191, 1.40822, 1.40981, 1.41905, 1.4096, 1.41551, 1.40808, 1.41685]}, "backward-compute-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [5.76315, 1.31571, 1.31593, 1.31502, 1.31389, 1.32096, 1.31535, 1.31393, 1.31645, 1.31983, 1.31373, 1.31879, 1.31981, 1.31802, 1.31437, 1.31804, 1.3168, 1.3164, 1.31781, 1.31891, 1.31627, 1.31955, 1.31518, 1.32254, 1.32375, 1.31999, 1.31794, 1.32051, 1.32225, 1.32201, 1.32279, 1.32113, 1.32401, 1.32399, 1.32517, 1.32129, 1.32334, 1.32013, 1.32408, 1.32339, 1.32077, 1.32325, 1.32393, 1.32691, 1.3248, 1.32346, 1.32319, 1.32546, 1.32574, 1.32432, 1.32506, 1.32316, 1.32102, 1.32498, 1.31925, 1.32089, 1.31762, 1.32259, 1.32419, 1.3238, 1.3311, 1.31611, 1.31766, 1.31858, 1.31753, 1.31906, 1.32287, 1.32538, 1.32481, 1.32145, 1.32464, 1.32198, 1.3244, 1.32137, 1.31992, 1.31987, 1.32194, 1.31437, 1.3176, 1.31699, 1.31617, 1.31875, 1.32414, 1.32452, 1.31883, 1.32118, 1.32409, 1.32097, 1.32779, 1.31828, 1.31626, 1.32197, 1.32549, 1.32434, 1.32206, 1.31897, 1.31696, 1.32081, 1.31817, 1.32008, 1.32093, 1.32034, 1.32057, 1.3194, 1.31784, 1.32222, 1.31761, 1.31937, 1.32438, 1.32014, 1.31951, 1.31748, 1.31751, 1.31806, 1.31789, 1.32196, 1.32358, 1.31991, 1.31901, 1.32185, 1.32603, 1.32323, 1.32207, 1.31786, 1.31601, 1.32365, 1.32045, 1.31939, 1.32039, 1.31927, 1.31562, 1.32046, 1.31813, 1.32192, 1.31787, 1.31521, 1.33243, 1.31979, 1.3209, 1.32524, 1.32073, 1.31982, 1.31934, 1.32334, 1.31999, 1.32008, 1.32149, 1.32088, 1.31917, 1.3216, 1.3281, 1.32441, 1.33089, 1.32051, 1.31858, 1.32678, 1.32537, 1.3342, 1.32893, 1.32448, 1.32645, 1.32391, 1.3234, 1.32535, 1.32031, 1.32412, 1.3238, 1.32447, 1.32647, 1.32957, 1.32786, 1.3237, 1.32721, 1.32175, 1.32877, 1.32685, 1.32128, 1.32422, 1.32282, 1.32689, 1.33079, 1.33206, 1.32599, 1.32533, 1.32086, 1.32573, 1.32664, 1.31836, 1.32782, 1.32904, 1.32799, 1.32601, 1.32546, 1.32741, 1.32429, 1.32809, 1.32601, 1.32401, 1.32374, 1.32751, 1.32317, 1.32231, 1.32071, 1.32437, 1.32903, 1.3223, 1.32056, 1.32302, 1.32275, 1.32175, 1.31913, 1.32111, 1.3226, 1.32065, 1.32224, 1.31853, 1.32253, 1.32127, 1.3209, 1.31926, 1.31964, 1.3227, 1.32157, 1.32205, 1.3223, 1.31767, 1.31875, 1.31811, 1.3211, 1.3162, 1.32259, 1.3172, 1.31878, 1.31747, 1.32111, 1.31966, 1.31682, 1.32112, 1.31521, 1.31669, 1.31901, 1.32814, 1.32216, 1.32442, 1.32313, 1.32151, 1.3243, 1.3203, 1.31897, 1.32073, 1.32493, 1.3246, 1.31844, 1.3284, 1.32684, 1.31608, 1.32499, 1.31768, 1.31464, 1.31825, 1.31743, 1.32077, 1.31974, 1.32195, 1.32195, 1.32016, 1.32093, 1.32005, 1.32407, 1.31906, 1.32446, 1.32365, 1.32141, 1.32093, 1.33319, 1.32834, 1.32237, 1.32312, 1.31793, 1.32722, 1.31541, 1.322, 1.3218, 1.31794, 1.31628, 1.31547, 1.32499, 1.31709, 1.317, 1.32129, 1.32324, 1.3231, 1.32155, 1.32292, 1.32269, 1.32156, 1.31852, 1.31872, 1.31758, 1.32143, 1.32104, 1.32353, 1.32012, 1.32147, 1.32263, 1.32328, 1.32548, 1.32214, 1.32307, 1.32574, 1.32903, 1.3278, 1.32381, 1.32116, 1.32264, 1.32367, 1.31807, 1.32574, 1.32105, 1.32208, 1.32432, 1.32324, 1.32004, 1.32242, 1.32161, 1.32001, 1.32057, 1.31875, 1.32152, 1.32786, 1.32575, 1.32357, 1.3226, 1.31921, 1.32595, 1.31832, 1.31725, 1.32287, 1.32418, 1.32617, 1.32128, 1.32384, 1.31932, 1.32117, 1.3209, 1.32292, 1.32281, 1.33147, 1.32181, 1.32357, 1.32241, 1.32062, 1.32002, 1.32089, 1.32929, 1.3178, 1.31998, 1.32166, 1.32279, 1.32038, 1.31604, 1.321, 1.31845, 1.31976, 1.32049, 1.32671, 1.30205, 1.30334, 1.30428, 1.30688, 1.30105, 1.306, 1.30598, 1.30505, 1.30135, 1.30452, 1.30666, 1.30463, 1.30387, 1.30213, 1.30721, 1.30426, 1.30532, 1.30358, 1.30289, 1.30331, 1.30072, 1.30374, 1.30623, 1.30837, 1.30441, 1.30441, 1.30428, 1.30182, 1.29924, 1.31777, 1.31621, 1.32106, 1.31759, 1.32273]}, "batch-generator-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [4.17805, 0.02532, 0.02443, 0.0259, 0.02446, 0.02433, 0.02525, 0.02434, 0.02571, 0.02834, 0.02652, 0.02646, 0.02518, 0.02481, 0.0279, 0.02807, 0.0266, 0.02845, 0.0313, 0.02866, 0.02895, 0.02709, 0.02883, 0.02971, 0.03025, 0.02951, 0.02896, 0.03006, 0.03215, 0.0295, 0.03352, 0.02739, 0.02956, 0.02814, 0.02868, 0.02699, 0.02842, 0.03193, 0.02797, 0.02967, 0.0318, 0.02963, 0.02835, 0.02797, 0.02797, 0.03173, 0.02956, 0.02665, 0.02908, 0.02921, 0.02665, 0.02893, 0.02866, 0.02772, 0.02944, 0.03233, 0.02893, 0.03067, 0.03096, 0.02981, 0.02909, 0.02673, 0.02735, 0.03183, 0.03003, 0.02892, 0.02792, 0.03046, 0.02823, 0.03032, 0.03123, 0.02966, 0.03045, 0.03048, 0.03141, 0.03097, 0.02999, 0.03135, 0.0285, 0.02735, 0.02803, 0.02831, 0.02764, 0.03034, 0.02971, 0.02926, 0.02972, 0.02952, 0.03075, 0.03009, 0.02964, 0.02882, 0.03045, 0.02898, 0.02803, 0.02824, 0.02708, 0.02867, 0.0342, 0.03142, 0.03184, 0.03236, 0.03305, 0.03116, 0.02898, 0.03026, 0.02775, 0.02983, 0.03023, 0.02832, 0.03086, 0.02777, 0.03086, 0.0307, 0.02887, 0.03065, 0.03095, 0.02937, 0.02703, 0.02981, 0.02895, 0.03324, 0.02658, 0.02662, 0.02448, 0.02629, 0.02739, 0.0271, 0.02673, 0.0253, 0.02683, 0.02718, 0.02671, 0.0276, 0.02593, 0.02704, 0.0285, 0.02845, 0.02811, 0.02883, 0.03435, 0.03167, 0.03261, 0.03235, 0.03414, 0.03091, 0.03163, 0.02955, 0.03106, 0.03182, 0.03113, 0.03157, 0.03216, 0.03397, 0.03111, 0.02941, 0.02991, 0.02875, 0.03204, 0.02798, 0.02854, 0.03038, 0.02648, 0.02916, 0.02799, 0.02855, 0.02792, 0.0274, 0.02603, 0.02879, 0.0292, 0.02864, 0.02841, 0.02759, 0.02946, 0.02947, 0.02937, 0.02887, 0.0288, 0.02812, 0.02927, 0.02796, 0.02893, 0.02755, 0.0266, 0.02892, 0.02827, 0.02802, 0.02761, 0.0284, 0.03055, 0.02773, 0.02955, 0.02851, 0.02789, 0.02748, 0.0272, 0.02827, 0.02809, 0.02816, 0.40686, 0.0267, 0.02546, 0.02555, 0.02624, 0.02523, 0.02567, 0.0279, 0.02868, 0.02572, 0.02653, 0.02383, 0.02613, 0.02506, 0.0243, 0.02629, 0.02418, 0.02447, 0.02537, 0.02552, 0.02379, 0.02344, 0.02378, 0.02314, 0.02354, 0.02382, 0.02379, 0.02659, 0.02476, 0.02631, 0.02468, 0.02598, 0.02324, 0.02455, 0.0251, 0.02405, 0.02442, 0.02377, 0.02361, 0.02478, 0.02379, 0.02477, 0.02439, 0.02295, 0.02552, 0.02359, 0.02286, 0.02462, 0.02531, 0.03164, 0.0315, 0.03143, 0.03142, 0.03168, 0.03139, 0.03399, 0.03158, 0.03159, 0.03346, 0.03175, 0.03166, 0.03151, 0.03142, 0.03168, 0.0317, 0.03164, 0.03167, 0.03175, 0.03163, 0.03326, 0.03172, 0.03141, 0.03173, 0.0333, 0.03168, 0.03167, 0.03183, 0.03165, 0.03174, 0.03408, 0.03301, 0.0256, 0.02643, 0.03, 0.02476, 0.02404, 0.02678, 0.02289, 0.02528, 0.02495, 0.02516, 0.02679, 0.02413, 0.0253, 0.02382, 0.02499, 0.02624, 0.02366, 0.02553, 0.02515, 0.02467, 0.02526, 0.02422, 0.02599, 0.02234, 0.02467, 0.02456, 0.02225, 0.02224, 0.02432, 0.02273, 0.02327, 0.02338, 0.02313, 0.02296, 0.02582, 0.02257, 0.02356, 0.02376, 0.02243, 0.02388, 0.02445, 0.02411, 0.02604, 0.02457, 0.02385, 0.02605, 0.02638, 0.02472, 0.02454, 0.02557, 0.02531, 0.02518, 0.02578, 0.02479, 0.02654, 0.02415, 0.02363, 0.02446, 0.02512, 0.02364, 0.02344, 0.0248, 0.02395, 0.02369, 0.02275, 0.0266, 0.02372, 0.02937, 0.02788, 0.02818, 0.02749, 0.0294, 0.02843, 0.02616, 0.02729, 0.02853, 0.02827, 0.02973, 0.02869, 0.02904, 0.02745, 0.02987, 0.02735, 0.02842, 0.02783, 0.02939, 0.02873, 0.02953, 0.02571, 0.02937, 0.02728, 0.03078, 0.02725, 0.02698, 0.02961, 0.02757, 0.02692, 0.02716, 0.02762, 0.02805, 0.02617, 0.02782, 0.02921, 0.02637, 0.02679, 0.02731, 0.02744, 0.02767, 0.02735, 0.02706, 0.02798, 0.02659, 0.02462, 0.02353, 0.02612, 0.02398, 0.02999, 0.02748, 0.02836]}, "forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.80244, 0.02327, 0.02357, 0.02418, 0.02403, 0.02416, 0.02299, 0.02437, 0.02654, 0.02645, 0.02351, 0.02322, 0.02321, 0.02333, 0.02356, 0.02407, 0.02284, 0.02336, 0.02305, 0.02309, 0.02437, 0.02382, 0.02371, 0.02295, 0.0237, 0.02304, 0.02301, 0.02347, 0.02339, 0.02268, 0.02304, 0.02357, 0.02381, 0.02335, 0.02274, 0.02277, 0.02379, 0.02387, 0.02489, 0.023, 0.02356, 0.02397, 0.02382, 0.0233, 0.02371, 0.02556, 0.02297, 0.02329, 0.02457, 0.02391, 0.02309, 0.02372, 0.02319, 0.02317, 0.02516, 0.02376, 0.02587, 0.02328, 0.02429, 0.02353, 0.02342, 0.02529, 0.02337, 0.02294, 0.02608, 0.0263, 0.02427, 0.02258, 0.02358, 0.02315, 0.02427, 0.02338, 0.02373, 0.02348, 0.02312, 0.02582, 0.02644, 0.02485, 0.02527, 0.02355, 0.02335, 0.0233, 0.02482, 0.02366, 0.02378, 0.02279, 0.02307, 0.02344, 0.02368, 0.02351, 0.02442, 0.023, 0.02371, 0.02324, 0.02397, 0.02339, 0.02331, 0.02303, 0.02316, 0.02451, 0.02588, 0.02323, 0.02313, 0.02372, 0.02372, 0.02396, 0.02313, 0.02377, 0.02325, 0.02357, 0.0239, 0.02373, 0.02305, 0.02327, 0.02337, 0.02558, 0.02412, 0.024, 0.02298, 0.02346, 0.02341, 0.02499, 0.02595, 0.02356, 0.02359, 0.02334, 0.02429, 0.02386, 0.02382, 0.02371, 0.02386, 0.02339, 0.02348, 0.02376, 0.02405, 0.0237, 0.02364, 0.02322, 0.02388, 0.02466, 0.02377, 0.02381, 0.02312, 0.02337, 0.02587, 0.0234, 0.02326, 0.02514, 0.02305, 0.02396, 0.02437, 0.02598, 0.02368, 0.02533, 0.02665, 0.0236, 0.02411, 0.02378, 0.02367, 0.02564, 0.02335, 0.02437, 0.02359, 0.02359, 0.02322, 0.02273, 0.02363, 0.02409, 0.02377, 0.02329, 0.02348, 0.02525, 0.02415, 0.02404, 0.02377, 0.02324, 0.02347, 0.02488, 0.02554, 0.02377, 0.02292, 0.02356, 0.02386, 0.0231, 0.024, 0.02405, 0.02445, 0.02374, 0.0233, 0.02593, 0.02463, 0.02393, 0.02351, 0.02352, 0.02404, 0.02313, 0.02358, 0.023, 0.02347, 0.02311, 0.0184, 0.02425, 0.02279, 0.02306, 0.02344, 0.02342, 0.0236, 0.02302, 0.02314, 0.02343, 0.02401, 0.02356, 0.02333, 0.02337, 0.0239, 0.0232, 0.02319, 0.02315, 0.02311, 0.02332, 0.02322, 0.02374, 0.0239, 0.02339, 0.02406, 0.02358, 0.02348, 0.02325, 0.02315, 0.02296, 0.02357, 0.02349, 0.02309, 0.02301, 0.02331, 0.02297, 0.0231, 0.02275, 0.0228, 0.02389, 0.02406, 0.02363, 0.02344, 0.02354, 0.02484, 0.02357, 0.02352, 0.02299, 0.02319, 0.02863, 0.02719, 0.02688, 0.0269, 0.02723, 0.02735, 0.02746, 0.02726, 0.02718, 0.02716, 0.02769, 0.02662, 0.02726, 0.0267, 0.02696, 0.02791, 0.0283, 0.03114, 0.02684, 0.02732, 0.02729, 0.02733, 0.02819, 0.02627, 0.02696, 0.02662, 0.02733, 0.02779, 0.02734, 0.02763, 0.02837, 0.02759, 0.0243, 0.02432, 0.02438, 0.02516, 0.02609, 0.02417, 0.02421, 0.02474, 0.02395, 0.02467, 0.02473, 0.02401, 0.02443, 0.02436, 0.02298, 0.02466, 0.02296, 0.02367, 0.02539, 0.02323, 0.02331, 0.02342, 0.02489, 0.02322, 0.02363, 0.02342, 0.02351, 0.02406, 0.02499, 0.02419, 0.02319, 0.02365, 0.02437, 0.02332, 0.02567, 0.02334, 0.02317, 0.02303, 0.02331, 0.02511, 0.02368, 0.02344, 0.02325, 0.0228, 0.02289, 0.02343, 0.02335, 0.0232, 0.02328, 0.02284, 0.0232, 0.02311, 0.02333, 0.02283, 0.02447, 0.02426, 0.02348, 0.02331, 0.02357, 0.02346, 0.02327, 0.02297, 0.0251, 0.02286, 0.0231, 0.02375, 0.02341, 0.0236, 0.0242, 0.02362, 0.02329, 0.02326, 0.02314, 0.02334, 0.02339, 0.02303, 0.02333, 0.02388, 0.02393, 0.02465, 0.02337, 0.02531, 0.02298, 0.02289, 0.02335, 0.02349, 0.02508, 0.02386, 0.02407, 0.0236, 0.02345, 0.02369, 0.02324, 0.02345, 0.02571, 0.02352, 0.02371, 0.02373, 0.02446, 0.02392, 0.02353, 0.02392, 0.02388, 0.02532, 0.02461, 0.02311, 0.02351, 0.02348, 0.02325, 0.02355, 0.02471, 0.02432, 0.0244, 0.02494, 0.02414, 0.02399, 0.02358, 0.02344, 0.02423]}, "forward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [2.84466, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00013, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00013, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00014, 0.00012, 0.00012, 0.00011, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00011, 0.00011, 0.00021, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00011, 0.00012, 0.00011, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00011, 0.00013, 0.00012, 0.00012, 0.00011, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00011, 0.00012, 0.00011, 0.00012, 0.00011, 0.00012, 0.00011, 0.00011, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00016, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00014, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00014, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00014, 0.00014, 0.00016, 0.00015, 0.0002, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00015, 0.00015, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00011, 0.00013, 0.00014, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00013, 0.00012, 0.00011, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00014, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00013, 0.00013, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00012, 0.00013, 0.00012, 0.00013, 0.00014, 0.00012, 0.00013, 0.00012]}, "backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.02202, 0.02306, 0.02274, 0.02305, 0.02218, 0.02282, 0.02254, 0.02256, 0.02256, 0.02201, 0.02227, 0.02236, 0.02184, 0.02219, 0.02311, 0.02279, 0.0224, 0.02326, 0.0223, 0.0226, 0.02262, 0.02192, 0.02207, 0.02234, 0.0225, 0.02331, 0.02364, 0.02244, 0.02259, 0.02244, 0.02307, 0.0232, 0.02442, 0.02498, 0.02229, 0.0228, 0.02468, 0.02377, 0.02241, 0.02261, 0.02253, 0.02261, 0.02234, 0.02253, 0.02252, 0.02275, 0.02272, 0.02219, 0.02235, 0.02245, 0.02519, 0.02285, 0.02297, 0.02413, 0.02237, 0.02293, 0.0228, 0.02258, 0.02227, 0.02742, 0.02319, 0.02305, 0.02286, 0.02291, 0.02288, 0.02328, 0.02324, 0.02362, 0.02461, 0.02229, 0.02295, 0.02276, 0.0234, 0.02322, 0.02241, 0.02264, 0.02302, 0.0234, 0.02233, 0.02257, 0.02316, 0.02277, 0.02753, 0.02283, 0.02254, 0.02283, 0.0218, 0.02217, 0.02286, 0.02257, 0.0228, 0.0227, 0.02081, 0.0228, 0.02621, 0.02311, 0.02273, 0.0228, 0.02247, 0.0229, 0.02301, 0.02246, 0.02269, 0.02282, 0.02255, 0.02285, 0.02311, 0.0227, 0.02235, 0.02252, 0.02338, 0.02261, 0.02365, 0.02278, 0.02199, 0.0226, 0.02251, 0.02252, 0.0226, 0.02281, 0.02411, 0.02301, 0.02114, 0.02254, 0.0225, 0.02292, 0.02388, 0.02719, 0.02225, 0.02241, 0.02306, 0.02278, 0.02254, 0.02221, 0.02262, 0.02523, 0.02237, 0.0224, 0.0224, 0.02234, 0.02308, 0.02372, 0.02327, 0.02279, 0.02316, 0.02344, 0.02202, 0.02286, 0.02663, 0.02281, 0.0234, 0.02273, 0.02221, 0.02282, 0.02274, 0.02532, 0.02225, 0.02195, 0.02261, 0.02257, 0.02265, 0.02262, 0.02232, 0.023, 0.02283, 0.02245, 0.02247, 0.0238, 0.02512, 0.02216, 0.0226, 0.02248, 0.02442, 0.02357, 0.02268, 0.02197, 0.02269, 0.02234, 0.02252, 0.02254, 0.02296, 0.02323, 0.02487, 0.02507, 0.02281, 0.02321, 0.01969, 0.02212, 0.02259, 0.02247, 0.02216, 0.02227, 0.02334, 0.02365, 0.02317, 0.02332, 0.02536, 0.02524, 0.02256, 0.02014, 0.02168, 0.02553, 0.02195, 0.02188, 0.02265, 0.02181, 0.02201, 0.02208, 0.02185, 0.02258, 0.02179, 0.02208, 0.02184, 0.02172, 0.02131, 0.02178, 0.02181, 0.02153, 0.02161, 0.02189, 0.02179, 0.02189, 0.02152, 0.02237, 0.01986, 0.02159, 0.02198, 0.02172, 0.02198, 0.02071, 0.0218, 0.02168, 0.02163, 0.02171, 0.02187, 0.02247, 0.0254, 0.02003, 0.02151, 0.02205, 0.02189, 0.02196, 0.02212, 0.02259, 0.02231, 0.02186, 0.0214, 0.02189, 0.02217, 0.02191, 0.02194, 0.02196, 0.02437, 0.0235, 0.02355, 0.02243, 0.02206, 0.02142, 0.02199, 0.02213, 0.02157, 0.02436, 0.02121, 0.02302, 0.0223, 0.02427, 0.02238, 0.02253, 0.01864, 0.02424, 0.02409, 0.0246, 0.02317, 0.02239, 0.02214, 0.02205, 0.022, 0.02349, 0.02219, 0.02161, 0.022, 0.02154, 0.02174, 0.0218, 0.02159, 0.02209, 0.022, 0.02163, 0.02288, 0.02366, 0.0234, 0.02153, 0.02198, 0.0241, 0.02181, 0.02185, 0.02225, 0.0216, 0.02178, 0.02096, 0.02214, 0.02076, 0.0219, 0.02303, 0.02184, 0.02342, 0.01921, 0.02176, 0.02172, 0.02189, 0.0219, 0.02192, 0.02085, 0.02133, 0.02429, 0.02384, 0.0242, 0.0195, 0.02178, 0.02175, 0.02146, 0.02171, 0.02168, 0.02164, 0.02417, 0.02331, 0.02162, 0.02199, 0.02187, 0.02172, 0.02155, 0.02173, 0.02177, 0.02367, 0.02387, 0.02186, 0.02165, 0.0215, 0.02171, 0.02193, 0.02169, 0.02399, 0.02207, 0.02179, 0.02207, 0.02217, 0.02226, 0.02196, 0.02201, 0.02182, 0.02159, 0.02152, 0.02173, 0.02179, 0.02146, 0.02161, 0.02161, 0.02191, 0.02365, 0.02194, 0.02182, 0.02252, 0.0217, 0.02184, 0.02214, 0.0207, 0.02212, 0.02196, 0.02227, 0.0219, 0.02213, 0.02179, 0.02192, 0.02063, 0.02245, 0.02495, 0.02207, 0.02234, 0.0219, 0.02176, 0.02221, 0.02198, 0.02398, 0.02453, 0.02261, 0.02208, 0.02163, 0.02214, 0.02159, 0.02483, 0.02236, 0.0221, 0.02206, 0.02218, 0.02227, 0.02233, 0.02258, 0.02182, 0.02191, 0.02178]}, "backward-send-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00019, 0.00019, 0.00018, 0.00017, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00018, 0.00019, 0.00019, 0.00017, 0.00018, 0.00017, 0.00019, 0.00018, 0.00022, 0.0002, 0.00018, 0.00019, 0.00016, 0.00017, 0.00018, 0.00019, 0.00018, 0.00017, 0.00018, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00018, 0.00019, 0.00018, 0.0002, 0.00017, 0.0002, 0.00018, 0.00019, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00019, 0.00018, 0.00016, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00019, 0.00019, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00021, 0.00019, 0.00019, 0.00018, 0.00016, 0.00018, 0.00019, 0.00019, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.0002, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00022, 0.00018, 0.00018, 0.0002, 0.00018, 0.00019, 0.00019, 0.00018, 0.00019, 0.00019, 0.00018, 0.00016, 0.00019, 0.00018, 0.00018, 0.00032, 0.00019, 0.00018, 0.00018, 0.00019, 0.00019, 0.00019, 0.00018, 0.00017, 0.00019, 0.00016, 0.00016, 0.00017, 0.00019, 0.00019, 0.00018, 0.00018, 0.00019, 0.00018, 0.00017, 0.00018, 0.00019, 0.00016, 0.00018, 0.00018, 0.00018, 0.00019, 0.00019, 0.00018, 0.00026, 0.00019, 0.00018, 0.00018, 0.00017, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00022, 0.00018, 0.00019, 0.00019, 0.00016, 0.00019, 0.00019, 0.00019, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00017, 0.00018, 0.00018, 0.00027, 0.00018, 0.00019, 0.00018, 0.00019, 0.00018, 0.00018, 0.00019, 0.00021, 0.00018, 0.00019, 0.00016, 0.00019, 0.00016, 0.00019, 0.00023, 0.00017, 0.00016, 0.00018, 0.00019, 0.00019, 0.00019, 0.00021, 0.00016, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00019, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00018, 0.00019, 0.00021, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00023, 0.00018, 0.00016, 0.00019, 0.00018, 0.00016, 0.00018, 0.00019, 0.00017, 0.00019, 0.00018, 0.00016, 0.00017, 0.00018, 0.00018, 0.00016, 0.00018, 0.00017, 0.00016, 0.00019, 0.00017, 0.00018, 0.00018, 0.00017, 0.00018, 0.00017, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00025, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00016, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00017, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00019, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00016, 0.00019, 0.00017, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00018, 0.00017, 0.00016, 0.00018, 0.00018, 0.00018, 0.00021, 0.00016, 0.00016, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00016, 0.00016, 0.00018, 0.00017, 0.00019, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00019, 0.00018, 0.00017, 0.00017, 0.00018, 0.00017, 0.00018, 0.00018, 0.00017, 0.00018, 0.00019, 0.00018, 0.00016, 0.00019, 0.00018, 0.00018, 0.00018, 0.00016, 0.00018, 0.00018, 0.00018, 0.00018, 0.00017, 0.00018, 0.00016, 0.00018, 0.00019, 0.00018, 0.00018, 0.00016, 0.00016, 0.00017, 0.00021, 0.00016, 0.00018, 0.00018, 0.00017, 0.00018, 0.00018, 0.00018, 0.00018, 0.00018, 0.00019, 0.00018, 0.00017, 0.00017, 0.00018, 0.00017, 0.00018]}, "forward-send-backward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [7.26791, 0.08664, 0.09388, 0.09112, 0.08445, 0.09357, 0.09373, 0.09614, 0.09989, 0.10112, 0.08956, 0.08704, 0.09001, 0.09155, 0.09857, 0.09953, 0.0961, 0.10113, 0.10125, 0.11004, 0.10313, 0.09862, 0.10585, 0.10919, 0.10583, 0.10172, 0.10458, 0.10404, 0.1052, 0.09641, 0.10412, 0.09781, 0.09972, 0.10136, 0.10163, 0.09609, 0.09969, 0.10085, 0.10306, 0.10325, 0.10455, 0.10533, 0.1025, 0.09569, 0.09963, 0.11379, 0.10728, 0.10291, 0.10638, 0.1012, 0.09514, 0.10381, 0.10024, 0.10547, 0.10487, 0.11789, 0.11734, 0.11997, 0.113, 0.10597, 0.11163, 0.11506, 0.12069, 0.12521, 0.12131, 0.11375, 0.10345, 0.10129, 0.10181, 0.10088, 0.0947, 0.09723, 0.09642, 0.10255, 0.10466, 0.09713, 0.10564, 0.10312, 0.10025, 0.09561, 0.09512, 0.09519, 0.08816, 0.09549, 0.09265, 0.09294, 0.10255, 0.09939, 0.10544, 0.10344, 0.10858, 0.1088, 0.10697, 0.09761, 0.09215, 0.09749, 0.10389, 0.09421, 0.09597, 0.09688, 0.10356, 0.10031, 0.10358, 0.10022, 0.09494, 0.09521, 0.08777, 0.09024, 0.09559, 0.08704, 0.09044, 0.08853, 0.09387, 0.09487, 0.09496, 0.0917, 0.09224, 0.08543, 0.08296, 0.0931, 0.08686, 0.09041, 0.08634, 0.0838, 0.07721, 0.08382, 0.08905, 0.07994, 0.08964, 0.09067, 0.08724, 0.09031, 0.09142, 0.08955, 0.08642, 0.08734, 0.09313, 0.0892, 0.08811, 0.08748, 0.10918, 0.10445, 0.10103, 0.10406, 0.10336, 0.10399, 0.11053, 0.10502, 0.1058, 0.10377, 0.10177, 0.10263, 0.10865, 0.10227, 0.1032, 0.10523, 0.08465, 0.08812, 0.09221, 0.0869, 0.09106, 0.09518, 0.08366, 0.09187, 0.09167, 0.09065, 0.08392, 0.08171, 0.08992, 0.09232, 0.08837, 0.08382, 0.08792, 0.08609, 0.08649, 0.09183, 0.09528, 0.08861, 0.08269, 0.07853, 0.08798, 0.08353, 0.08436, 0.09088, 0.08495, 0.08552, 0.08561, 0.08913, 0.08612, 0.08093, 0.08731, 0.08686, 0.08376, 0.09109, 0.08222, 0.08599, 0.08546, 0.09351, 0.09605, 0.09994, 0.05805, 0.06314, 0.06773, 0.06769, 0.07278, 0.07311, 0.07124, 0.07502, 0.06435, 0.06762, 0.06901, 0.0791, 0.0778, 0.07332, 0.07358, 0.07456, 0.08054, 0.08433, 0.07505, 0.07588, 0.08407, 0.0787, 0.08207, 0.0796, 0.07151, 0.06957, 0.07132, 0.06499, 0.06604, 0.07296, 0.07397, 0.067, 0.07615, 0.07913, 0.07517, 0.07077, 0.07248, 0.07492, 0.07227, 0.07335, 0.0763, 0.07019, 0.07546, 0.07774, 0.07407, 0.0729, 0.07638, 0.07126, 0.07892, 0.09584, 0.09387, 0.09457, 0.09277, 0.0883, 0.08843, 0.09465, 0.09754, 0.09491, 0.09011, 0.08659, 0.08508, 0.08604, 0.09074, 0.08671, 0.08822, 0.08652, 0.10003, 0.09872, 0.09528, 0.09138, 0.09197, 0.09145, 0.09609, 0.09717, 0.09187, 0.08329, 0.07444, 0.08501, 0.09292, 0.07912, 0.09086, 0.06371, 0.06325, 0.06657, 0.06269, 0.0684, 0.06721, 0.07116, 0.07046, 0.0677, 0.06735, 0.06869, 0.06628, 0.06387, 0.06598, 0.06628, 0.06315, 0.07014, 0.06138, 0.06023, 0.06541, 0.06746, 0.07002, 0.07338, 0.06917, 0.06109, 0.06706, 0.07059, 0.07159, 0.07375, 0.08229, 0.07701, 0.07396, 0.07568, 0.07085, 0.07045, 0.06836, 0.06539, 0.0665, 0.07089, 0.0709, 0.06602, 0.0697, 0.07478, 0.0684, 0.0647, 0.0626, 0.06703, 0.06836, 0.06571, 0.07061, 0.07022, 0.0716, 0.06385, 0.06344, 0.05399, 0.06182, 0.0629, 0.06795, 0.07021, 0.06979, 0.06991, 0.07026, 0.06139, 0.06342, 0.06547, 0.06176, 0.06228, 0.07216, 0.07562, 0.07274, 0.07226, 0.08023, 0.07444, 0.04375, 0.0697, 0.07621, 0.07857, 0.07477, 0.07791, 0.08106, 0.08001, 0.07886, 0.07928, 0.08279, 0.07305, 0.08365, 0.08546, 0.08515, 0.08206, 0.08649, 0.09308, 0.09213, 0.08788, 0.08419, 0.0881, 0.09226, 0.08474, 0.08747, 0.08269, 0.08805, 0.08503, 0.08089, 0.08025, 0.07691, 0.07938, 0.07913, 0.08725, 0.08008, 0.08335, 0.0882, 0.08124, 0.08869, 0.08118, 0.08321, 0.08276, 0.07892, 0.08691, 0.07849, 0.08318]}, "backward-send-forward-recv-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3.02438, 0.02964, 0.02158, 0.02612, 0.02742, 0.02646, 0.02144, 0.01953, 0.02104, 0.01973, 0.0221, 0.02679, 0.02821, 0.0292, 0.02641, 0.02434, 0.02851, 0.02189, 0.02401, 0.02493, 0.02324, 0.02474, 0.02466, 0.01958, 0.02074, 0.02324, 0.02406, 0.02422, 0.02172, 0.02415, 0.02078, 0.02874, 0.02875, 0.02888, 0.03126, 0.03155, 0.0297, 0.0288, 0.03235, 0.02835, 0.02837, 0.02808, 0.02869, 0.03298, 0.03478, 0.02725, 0.02531, 0.02971, 0.0248, 0.02835, 0.03171, 0.02666, 0.02768, 0.0316, 0.11725, 0.02233, 0.01927, 0.01846, 0.02324, 0.0208, 0.02765, 0.02234, 0.02152, 0.02055, 0.0218, 0.02092, 0.02617, 0.02621, 0.02575, 0.02487, 0.02854, 0.02512, 0.02754, 0.02441, 0.02799, 0.02601, 0.02443, 0.02664, 0.02842, 0.02747, 0.02197, 0.02705, 0.0286, 0.02828, 0.03081, 0.02999, 0.03156, 0.02772, 0.02622, 0.02462, 0.02412, 0.02594, 0.02264, 0.03102, 0.02956, 0.02597, 0.02756, 0.03008, 0.02803, 0.02913, 0.02661, 0.02374, 0.02365, 0.02578, 0.02542, 0.03028, 0.03098, 0.02753, 0.02526, 0.02933, 0.02658, 0.02632, 0.02526, 0.02436, 0.02205, 0.02173, 0.02147, 0.02635, 0.02715, 0.01835, 0.02341, 0.02286, 0.02713, 0.03176, 0.03552, 0.02684, 0.02459, 0.03111, 0.02691, 0.02888, 0.02912, 0.02835, 0.02868, 0.0319, 0.02488, 0.02699, 0.02738, 0.02288, 0.03107, 0.03026, 0.02374, 0.02063, 0.02531, 0.02048, 0.02199, 0.02504, 0.01991, 0.03009, 0.02384, 0.02452, 0.02777, 0.02276, 0.02322, 0.02545, 0.02596, 0.02803, 0.03054, 0.03445, 0.02978, 0.02853, 0.02578, 0.02477, 0.03074, 0.02951, 0.03089, 0.03187, 0.02945, 0.03462, 0.02761, 0.03327, 0.03222, 0.03039, 0.03257, 0.02712, 0.02729, 0.02863, 0.02412, 0.02627, 0.03209, 0.03064, 0.02986, 0.02923, 0.03127, 0.02881, 0.03666, 0.03233, 0.03454, 0.03286, 0.03299, 0.03171, 0.03363, 0.03637, 0.03532, 0.02997, 0.03427, 0.03447, 0.03788, 0.03045, 0.02935, 0.02785, 0.06375, 0.04913, 0.04593, 0.04639, 0.04315, 0.04609, 0.04022, 0.04069, 0.0458, 0.04145, 0.04193, 0.03809, 0.03122, 0.0379, 0.04024, 0.03151, 0.03065, 0.03028, 0.03812, 0.03701, 0.03342, 0.03675, 0.03239, 0.0438, 0.03695, 0.0419, 0.04267, 0.04585, 0.04997, 0.04424, 0.04745, 0.04667, 0.04464, 0.03917, 0.03907, 0.03699, 0.04231, 0.03898, 0.04045, 0.03812, 0.0373, 0.04307, 0.03851, 0.03799, 0.04077, 0.0409, 0.04045, 0.04407, 0.0328, 0.02602, 0.03043, 0.0238, 0.02775, 0.03236, 0.02827, 0.02216, 0.02607, 0.02209, 0.02438, 0.02661, 0.02817, 0.0302, 0.02384, 0.02743, 0.03022, 0.02263, 0.02281, 0.02357, 0.02756, 0.02656, 0.02806, 0.02726, 0.02917, 0.02779, 0.04648, 0.03625, 0.03939, 0.03798, 0.03027, 0.03365, 0.03112, 0.0507, 0.05041, 0.0488, 0.0478, 0.04287, 0.04273, 0.03793, 0.04099, 0.0473, 0.04686, 0.04606, 0.04653, 0.04791, 0.0434, 0.04395, 0.04672, 0.03952, 0.04338, 0.05238, 0.05084, 0.0447, 0.04529, 0.04014, 0.04009, 0.04618, 0.03869, 0.04044, 0.04097, 0.04238, 0.03044, 0.04364, 0.04057, 0.03549, 0.03892, 0.03761, 0.03631, 0.04319, 0.04214, 0.04271, 0.04566, 0.04209, 0.0419, 0.03476, 0.04175, 0.03736, 0.04126, 0.04073, 0.04268, 0.04088, 0.03755, 0.04007, 0.0375, 0.03951, 0.04011, 0.04621, 0.04174, 0.04428, 0.03833, 0.03393, 0.03343, 0.03715, 0.03224, 0.0391, 0.03809, 0.0352, 0.04357, 0.04052, 0.02489, 0.02136, 0.02147, 0.01936, 0.01974, 0.01753, 0.1141, 0.01901, 0.02217, 0.02537, 0.01881, 0.01782, 0.01594, 0.01966, 0.01818, 0.02087, 0.02147, 0.02626, 0.01794, 0.01552, 0.01646, 0.01963, 0.01985, 0.02306, 0.02056, 0.01929, 0.0188, 0.02041, 0.01882, 0.01934, 0.01928, 0.01858, 0.01964, 0.01987, 0.02011, 0.01922, 0.01909, 0.02055, 0.01875, 0.02072, 0.02181, 0.02052, 0.01786, 0.01986, 0.01947, 0.02245, 0.01734, 0.01752, 0.01965, 0.02295, 0.02233, 0.01907]}, "layernorm-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [3e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05, 2e-05, 1e-05, 1e-05, 1e-05]}, "embedding-grads-all-reduce-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00057, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00034, 0.00022, 0.00024, 0.00022, 0.00026, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00026, 0.00026, 0.00025, 0.00022, 0.00025, 0.00022, 0.00022, 0.00024, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00026, 0.00022, 0.00025, 0.00022, 0.00023, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00021, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00026, 0.00025, 0.00025, 0.00021, 0.00022, 0.00022, 0.00022, 0.00025, 0.00025, 0.00025, 0.00022, 0.00023, 0.00022, 0.00022, 0.00023, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00025, 0.00021, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00025, 0.00025, 0.00025, 0.00022, 0.00033, 0.00022, 0.00022, 0.00023, 0.00025, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00026, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00026, 0.00022, 0.00026, 0.00025, 0.00024, 0.00025, 0.00022, 0.00025, 0.00022, 0.00022, 0.00026, 0.00025, 0.00022, 0.00022, 0.00022, 0.00025, 0.00025, 0.00025, 0.00025, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00024, 0.00023, 0.00022, 0.00023, 0.00022, 0.00021, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00025, 0.00024, 0.00022, 0.00024, 0.00022, 0.00025, 0.00022, 0.00022, 0.00026, 0.00025, 0.00024, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00025, 0.00022, 0.00026, 0.00022, 0.00022, 0.00022, 0.00022, 0.00027, 0.00022, 0.00025, 0.00022, 0.00026, 0.00025, 0.00021, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00026, 0.00022, 0.00021, 0.00026, 0.00025, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00026, 0.00025, 0.00021, 0.00022, 0.00026, 0.00025, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00021, 0.00021, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00025, 0.00025, 0.00025, 0.00022, 0.00022, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00024, 0.00024, 0.00024, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00025, 0.00025, 0.00022, 0.00021, 0.00021, 0.00023, 0.00021, 0.00021, 0.00025, 0.00021, 0.00021, 0.00025, 0.00022, 0.00021, 0.00025, 0.00022, 0.00021, 0.00021, 0.00025, 0.00021, 0.00021, 0.00021, 0.00025, 0.00025, 0.00022, 0.00022, 0.00021, 0.00025, 0.00021, 0.00021, 0.00021, 0.00021, 0.00021, 0.00021, 0.00022, 0.00022, 0.00021, 0.00021, 0.00021, 0.00022, 0.00021, 0.00022, 0.00022, 0.00022, 0.00024, 0.00021, 0.00022, 0.00022, 0.00024, 0.00021, 0.00025, 0.00021, 0.00025, 0.00021, 0.00025, 0.00022, 0.00021, 0.00021, 0.00021, 0.00025, 0.00023, 0.00021, 0.00021, 0.00025, 0.00021, 0.00021, 0.00022, 0.00025, 0.00021, 0.00021, 0.00022, 0.00022, 0.00021, 0.00021, 0.00022, 0.00025, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00022, 0.00021, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00025, 0.00022, 0.00021, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00033, 0.00022, 0.00021, 0.00022, 0.00022, 0.00022, 0.00021, 0.00024]}, "all-grads-sync-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.66214, 0.00023, 0.00022, 0.00023, 0.00028, 0.00028, 0.00027, 0.00028, 0.00025, 0.00023, 0.00024, 0.00023, 0.00023, 0.00023, 0.00024, 0.00023, 0.00023, 0.00024, 0.00023, 0.00023, 0.00023, 0.0003, 0.00028, 0.00028, 0.00034, 0.00028, 0.00028, 0.00028, 0.00028, 0.00022, 0.00026, 0.00023, 0.00022, 0.00028, 0.00032, 0.00023, 0.00028, 0.00023, 0.00028, 0.00022, 0.00022, 0.00028, 0.00023, 0.00037, 0.00023, 0.00023, 0.00028, 0.00028, 0.00023, 0.00022, 0.00024, 0.00024, 0.00022, 0.00022, 0.00029, 0.00023, 0.00023, 0.00029, 0.00023, 0.00023, 0.00028, 0.00023, 0.00029, 0.00023, 0.00027, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00028, 0.00022, 0.00024, 0.00024, 0.00034, 0.00036, 0.00026, 0.00027, 0.00028, 0.00023, 0.00024, 0.00024, 0.00028, 0.00028, 0.00028, 0.00025, 0.00023, 0.00028, 0.00027, 0.00022, 0.00023, 0.00029, 0.00022, 0.00024, 0.00027, 0.00023, 0.00029, 0.00024, 0.00028, 0.00028, 0.00028, 0.00028, 0.00023, 0.00028, 0.00023, 0.00023, 0.00028, 0.00028, 0.0003, 0.00023, 0.00027, 0.00025, 0.00023, 0.00023, 0.00028, 0.00024, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00023, 0.00028, 0.00027, 0.00023, 0.00023, 0.00029, 0.00023, 0.00023, 0.00029, 0.00028, 0.00028, 0.00028, 0.00024, 0.00028, 0.00024, 0.00023, 0.00025, 0.00026, 0.00029, 0.00028, 0.00028, 0.00028, 0.00028, 0.00028, 0.00023, 0.00023, 0.00023, 0.00024, 0.00023, 0.0003, 0.00024, 0.00028, 0.00028, 0.00023, 0.00023, 0.00022, 0.00027, 0.00023, 0.00028, 0.00024, 0.00024, 0.00023, 0.00023, 0.00023, 0.00028, 0.00022, 0.00029, 0.00029, 0.00028, 0.00022, 0.00024, 0.0003, 0.00025, 0.00028, 0.00023, 0.00022, 0.00028, 0.00024, 0.00029, 0.00029, 0.00028, 0.00025, 0.00028, 0.00029, 0.00028, 0.00029, 0.00029, 0.00023, 0.00028, 0.00028, 0.00028, 0.00024, 0.0003, 0.00028, 0.00025, 0.00028, 0.00025, 0.00023, 0.00023, 0.00023, 0.00023, 0.00028, 0.00023, 0.00028, 0.00028, 0.00022, 0.00028, 0.00022, 0.00029, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00028, 0.00022, 0.00023, 0.00022, 0.00028, 0.00022, 0.00023, 0.00027, 0.00022, 0.00024, 0.00022, 0.00028, 0.00022, 0.00022, 0.00022, 0.00027, 0.00022, 0.00022, 0.00028, 0.00028, 0.00022, 0.00023, 0.00022, 0.00022, 0.00028, 0.00024, 0.00028, 0.00022, 0.00022, 0.00022, 0.00027, 0.00022, 0.00024, 0.00024, 0.00023, 0.00028, 0.00022, 0.00028, 0.00022, 0.00028, 0.00028, 0.00023, 0.00025, 0.00025, 0.00035, 0.00023, 0.00023, 0.00028, 0.00024, 0.00025, 0.00028, 0.00023, 0.00023, 0.00023, 0.00028, 0.00025, 0.00022, 0.00029, 0.00023, 0.00023, 0.00022, 0.00022, 0.00024, 0.00027, 0.00027, 0.00028, 0.00022, 0.00022, 0.00025, 0.00022, 0.00022, 0.00028, 0.00021, 0.00027, 0.00021, 0.00023, 0.00023, 0.00021, 0.00022, 0.00021, 0.00028, 0.00027, 0.00027, 0.00028, 0.00022, 0.00027, 0.00023, 0.00022, 0.00022, 0.00024, 0.00027, 0.00028, 0.00027, 0.00022, 0.00022, 0.00027, 0.00022, 0.00027, 0.00022, 0.00023, 0.00022, 0.00021, 0.00021, 0.00022, 0.00022, 0.00027, 0.00024, 0.00027, 0.00023, 0.00022, 0.00021, 0.00021, 0.00021, 0.00028, 0.00022, 0.00023, 0.00022, 0.00028, 0.00023, 0.00027, 0.00022, 0.00028, 0.00023, 0.00028, 0.00021, 0.00023, 0.00022, 0.00022, 0.00027, 0.00022, 0.00027, 0.00034, 0.00021, 0.00023, 0.00021, 0.00023, 0.00022, 0.00022, 0.00028, 0.00025, 0.00023, 0.00023, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00028, 0.00022, 0.00022, 0.00022, 0.00028, 0.00022, 0.00022, 0.00022, 0.00028, 0.00021, 0.00029, 0.00022, 0.00022, 0.00022, 0.00022, 0.00022, 0.00023, 0.00022, 0.00023, 0.0003, 0.00022, 0.00023, 0.00022, 0.00022, 0.00022, 0.00022, 0.00024, 0.00022, 0.00022, 0.00028, 0.00022, 0.00022, 0.00024, 0.00022]}, "optimizer-copy-to-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00016, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00014, 0.00016, 0.00014, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00016, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00014, 0.00015, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00014, 0.00014, 0.0002, 0.00019, 0.00019, 0.00019, 0.00019, 0.00019, 0.00018, 0.0002, 0.0002, 0.00019, 0.00019, 0.00019, 0.0002, 0.00019, 0.0002, 0.00019, 0.00019, 0.00019, 0.00019, 0.0002, 0.00019, 0.00019, 0.00019, 0.00018, 0.0002, 0.00019, 0.0002, 0.00019, 0.00019, 0.00019, 0.00019, 0.00019, 0.00015, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00013, 0.00015, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00014, 0.00013, 0.00013, 0.00015, 0.00014, 0.00013, 0.00013, 0.00014, 0.00013, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00014, 0.00013, 0.00013, 0.00013, 0.00013, 0.00015, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00013, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00015, 0.00013, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014, 0.00014]}, "optimizer-clip-main-grad-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.52041, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00057, 0.00059, 0.00059, 0.00055, 0.00058, 0.00055, 0.00059, 0.00056, 0.00055, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00055, 0.00055, 0.00055, 0.00054, 0.00053, 0.00054, 0.00069, 0.00054, 0.00071, 0.00057, 0.00073, 0.00055, 0.00054, 0.00054, 0.00054, 0.00056, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00057, 0.00059, 0.00054, 0.00054, 0.00054, 0.00055, 0.00055, 0.00055, 0.00056, 0.00054, 0.00056, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00058, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.0007, 0.00055, 0.00055, 0.00055, 0.00056, 0.00056, 0.00056, 0.00054, 0.00054, 0.00056, 0.00057, 0.00054, 0.00054, 0.00056, 0.00054, 0.0006, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00053, 0.00053, 0.00058, 0.00049, 0.00054, 0.00048, 0.00055, 0.00054, 0.00055, 0.00054, 0.00057, 0.00054, 0.00057, 0.00069, 0.00054, 0.00055, 0.00048, 0.00054, 0.00048, 0.00048, 0.0005, 0.00056, 0.00055, 0.00054, 0.00055, 0.00054, 0.00054, 0.00048, 0.00055, 0.00054, 0.00055, 0.00058, 0.00054, 0.00054, 0.00054, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00058, 0.00055, 0.00054, 0.00054, 0.00055, 0.00053, 0.00054, 0.00055, 0.00054, 0.00054, 0.00054, 0.00055, 0.00048, 0.00054, 0.00054, 0.00055, 0.00054, 0.00056, 0.00056, 0.00054, 0.00054, 0.00054, 0.00057, 0.00054, 0.00054, 0.00055, 0.00054, 0.00056, 0.00056, 0.00054, 0.00055, 0.00055, 0.00054, 0.00054, 0.00048, 0.00054, 0.00056, 0.00055, 0.00054, 0.00058, 0.00054, 0.00054, 0.00054, 0.00054, 0.00057, 0.00066, 0.00058, 0.00056, 0.00055, 0.00055, 0.00055, 0.00055, 0.00058, 0.00055, 0.00055, 0.00054, 0.00054, 0.00054, 0.00054, 0.00071, 0.00055, 0.00054, 0.00054, 0.0006, 0.00054, 0.00053, 0.00056, 0.00054, 0.00053, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00054, 0.00056, 0.00053, 0.00053, 0.00053, 0.00054, 0.00056, 0.00054, 0.00054, 0.00053, 0.00053, 0.00053, 0.00054, 0.00055, 0.00053, 0.00054, 0.00053, 0.00054, 0.00057, 0.00054, 0.00054, 0.00054, 0.00054, 0.00053, 0.00056, 0.00054, 0.00056, 0.00053, 0.00054, 0.00065, 0.00054, 0.00053, 0.00054, 0.00054, 0.00055, 0.00054, 0.00054, 0.00055, 0.00072, 0.00073, 0.00073, 0.00074, 0.00073, 0.00072, 0.00071, 0.00072, 0.0008, 0.00072, 0.00072, 0.00072, 0.00072, 0.00072, 0.00073, 0.00116, 0.00072, 0.00072, 0.00073, 0.00073, 0.00074, 0.00072, 0.00072, 0.00072, 0.00073, 0.00075, 0.00077, 0.00072, 0.00072, 0.00072, 0.00072, 0.00072, 0.00054, 0.00053, 0.00059, 0.00053, 0.00053, 0.00052, 0.00053, 0.00053, 0.00055, 0.00053, 0.00052, 0.00053, 0.00054, 0.00053, 0.00055, 0.00053, 0.00052, 0.00052, 0.00053, 0.00055, 0.00053, 0.00057, 0.00053, 0.00053, 0.00055, 0.00052, 0.00054, 0.00052, 0.00053, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00053, 0.00053, 0.00052, 0.00054, 0.00056, 0.00052, 0.00052, 0.00052, 0.00053, 0.00054, 0.00054, 0.00053, 0.00052, 0.00055, 0.00052, 0.00057, 0.00052, 0.00053, 0.00053, 0.00053, 0.00055, 0.00053, 0.00052, 0.00052, 0.00053, 0.00052, 0.00055, 0.00052, 0.00053, 0.00053, 0.00052, 0.00054, 0.00054, 0.00058, 0.00051, 0.00054, 0.00053, 0.00053, 0.00053, 0.00056, 0.00056, 0.00054, 0.00053, 0.00054, 0.00055, 0.00053, 0.00054, 0.00057, 0.00054, 0.00056, 0.00054, 0.00055, 0.00054, 0.00053, 0.00053, 0.00053, 0.00054, 0.00055, 0.00053, 0.00054, 0.00055, 0.00055, 0.00068, 0.00053, 0.00053, 0.00054, 0.00053, 0.00059, 0.00054, 0.00057, 0.00053, 0.00054, 0.00056, 0.00054, 0.00056, 0.00059, 0.00054, 0.00066, 0.00053, 0.00053, 0.00053, 0.00053, 0.00056, 0.0007, 0.00055]}, "optimizer-count-zeros-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00377, 0.00267, 0.00263, 0.00264, 0.00263, 0.00264, 0.00267, 0.00265, 0.00264, 0.00265, 0.00266, 0.00266, 0.00264, 0.00267, 0.00266, 0.00265, 0.00263, 0.00266, 0.00263, 0.00264, 0.00264, 0.00264, 0.00264, 0.00262, 0.00264, 0.00265, 0.00265, 0.00264, 0.00279, 0.00265, 0.0029, 0.00265, 0.00467, 0.00274, 0.00266, 0.00265, 0.00264, 0.00264, 0.00264, 0.00267, 0.00265, 0.00263, 0.00264, 0.00264, 0.00264, 0.00265, 0.00264, 0.00264, 0.00266, 0.00265, 0.00272, 0.00265, 0.00266, 0.00265, 0.00264, 0.00266, 0.00266, 0.00265, 0.00266, 0.00277, 0.00266, 0.00267, 0.00266, 0.00266, 0.00266, 0.00265, 0.00264, 0.00266, 0.00269, 0.00259, 0.00261, 0.00261, 0.0026, 0.00263, 0.00275, 0.00259, 0.00263, 0.00262, 0.0026, 0.00262, 0.00262, 0.0026, 0.00273, 0.00262, 0.00261, 0.00261, 0.0026, 0.0026, 0.00262, 0.00262, 0.00259, 0.0026, 0.0026, 0.00292, 0.00276, 0.00261, 0.00262, 0.00262, 0.00262, 0.00261, 0.00261, 0.0026, 0.0026, 0.00261, 0.00292, 0.00264, 0.00266, 0.0026, 0.00263, 0.00261, 0.00259, 0.00261, 0.0026, 0.00261, 0.00259, 0.0026, 0.00261, 0.00262, 0.00261, 0.0026, 0.00264, 0.00262, 0.00288, 0.00263, 0.00258, 0.00261, 0.00266, 0.00274, 0.00261, 0.0026, 0.00263, 0.00261, 0.0026, 0.00262, 0.00262, 0.00261, 0.00262, 0.00262, 0.00261, 0.0026, 0.00268, 0.00264, 0.00265, 0.00266, 0.00266, 0.00265, 0.00272, 0.00264, 0.00278, 0.00265, 0.00266, 0.00266, 0.00267, 0.00264, 0.00264, 0.00272, 0.0026, 0.00261, 0.00261, 0.00261, 0.00262, 0.00262, 0.00263, 0.00261, 0.00262, 0.00259, 0.00261, 0.00262, 0.00269, 0.0026, 0.00262, 0.00262, 0.00261, 0.00262, 0.00261, 0.00261, 0.00263, 0.0026, 0.00262, 0.0026, 0.00263, 0.00262, 0.0034, 0.00265, 0.00259, 0.00259, 0.0026, 0.00261, 0.00261, 0.0026, 0.00277, 0.0026, 0.00262, 0.00261, 0.00264, 0.00261, 0.00263, 0.00268, 0.00261, 0.0026, 0.00239, 0.00238, 0.0024, 0.00237, 0.00238, 0.00237, 0.00239, 0.00237, 0.0024, 0.0024, 0.00243, 0.00239, 0.0024, 0.0024, 0.00238, 0.00241, 0.00242, 0.00239, 0.00246, 0.00242, 0.0024, 0.00238, 0.00238, 0.00239, 0.00239, 0.00239, 0.00239, 0.0024, 0.0024, 0.00239, 0.00239, 0.00244, 0.00238, 0.00237, 0.00238, 0.0024, 0.00242, 0.00238, 0.00238, 0.00241, 0.00268, 0.00241, 0.00241, 0.00239, 0.00242, 0.00238, 0.00241, 0.00243, 0.00467, 0.00362, 0.00363, 0.0036, 0.00366, 0.00361, 0.00362, 0.00363, 0.00361, 0.00375, 0.00372, 0.00364, 0.0036, 0.00364, 0.00361, 0.00361, 0.00363, 0.00364, 0.00364, 0.00363, 0.00364, 0.00363, 0.00387, 0.00363, 0.00364, 0.00363, 0.00362, 0.00364, 0.00362, 0.00361, 0.00361, 0.00362, 0.00365, 0.00238, 0.00239, 0.00237, 0.0024, 0.0024, 0.00237, 0.00239, 0.00239, 0.00236, 0.00239, 0.00239, 0.00239, 0.00237, 0.00241, 0.00242, 0.00243, 0.00239, 0.0024, 0.00238, 0.00239, 0.00239, 0.00237, 0.00239, 0.00243, 0.00239, 0.00243, 0.00238, 0.00238, 0.00238, 0.00239, 0.00236, 0.0024, 0.00241, 0.00237, 0.00241, 0.0024, 0.00241, 0.00239, 0.00237, 0.0024, 0.00239, 0.0024, 0.00239, 0.00237, 0.00241, 0.00239, 0.00237, 0.00237, 0.0024, 0.00239, 0.00238, 0.00238, 0.0024, 0.00254, 0.00238, 0.00239, 0.00238, 0.00238, 0.00239, 0.00238, 0.00243, 0.00239, 0.00239, 0.00245, 0.00239, 0.00238, 0.00238, 0.00263, 0.00238, 0.00243, 0.00236, 0.00238, 0.00238, 0.00237, 0.00238, 0.00239, 0.0026, 0.00242, 0.0024, 0.0024, 0.0024, 0.0024, 0.00238, 0.00238, 0.00243, 0.00242, 0.0024, 0.00239, 0.0024, 0.0024, 0.00239, 0.00243, 0.00238, 0.0024, 0.00237, 0.00237, 0.00297, 0.0024, 0.0024, 0.00238, 0.00239, 0.00241, 0.00238, 0.00239, 0.00237, 0.00239, 0.00239, 0.00273, 0.00252, 0.00238, 0.00239, 0.00239, 0.00238, 0.00236, 0.0024, 0.0024, 0.00241, 0.00253, 0.00238]}, "optimizer-inner-step-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0039, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00044, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00046, 0.00047, 0.00045, 0.00046, 0.00045, 0.00046, 0.00059, 0.00046, 0.00046, 0.00045, 0.00046, 0.00062, 0.00046, 0.00061, 0.00045, 0.00047, 0.00046, 0.00045, 0.00046, 0.00045, 0.00045, 0.00047, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00052, 0.00045, 0.00045, 0.00046, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00045, 0.00045, 0.00047, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00045, 0.00046, 0.00046, 0.00045, 0.00053, 0.00046, 0.00045, 0.00046, 0.00046, 0.00047, 0.00045, 0.00045, 0.00054, 0.00045, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00064, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00049, 0.00047, 0.00047, 0.00046, 0.00048, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00048, 0.00046, 0.00047, 0.00046, 0.00047, 0.00059, 0.00048, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00055, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00046, 0.00048, 0.00047, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00049, 0.00047, 0.00046, 0.00047, 0.00046, 0.00048, 0.00045, 0.00045, 0.00046, 0.00046, 0.00047, 0.00046, 0.00045, 0.00045, 0.00045, 0.00047, 0.00046, 0.00047, 0.00063, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00047, 0.00045, 0.00048, 0.00046, 0.00046, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00046, 0.00046, 0.00045, 0.00049, 0.00046, 0.00048, 0.00045, 0.00047, 0.00057, 0.00045, 0.00047, 0.00045, 0.00046, 0.00047, 0.00045, 0.00046, 0.00051, 0.00059, 0.00058, 0.0006, 0.00059, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00058, 0.00058, 0.00058, 0.00059, 0.00058, 0.00059, 0.00059, 0.00058, 0.00059, 0.00059, 0.00059, 0.00061, 0.00059, 0.00058, 0.00058, 0.0006, 0.00059, 0.00058, 0.00058, 0.00059, 0.0006, 0.0006, 0.0006, 0.00045, 0.00045, 0.00045, 0.00043, 0.00044, 0.00045, 0.00043, 0.00045, 0.00043, 0.00045, 0.00043, 0.00044, 0.00045, 0.00044, 0.00044, 0.00044, 0.00044, 0.00044, 0.00044, 0.00045, 0.00043, 0.00043, 0.00044, 0.00061, 0.00046, 0.00045, 0.00043, 0.00045, 0.00043, 0.00044, 0.00044, 0.00045, 0.00044, 0.00044, 0.0006, 0.00044, 0.00044, 0.00044, 0.00044, 0.00045, 0.00042, 0.00043, 0.00043, 0.00043, 0.00045, 0.00045, 0.00044, 0.00046, 0.00044, 0.00044, 0.00043, 0.00043, 0.00047, 0.00043, 0.00043, 0.00044, 0.00043, 0.00044, 0.00044, 0.00043, 0.00045, 0.00044, 0.00044, 0.00044, 0.00043, 0.00044, 0.00044, 0.00045, 0.00045, 0.00044, 0.00045, 0.00045, 0.00044, 0.00046, 0.00044, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00046, 0.00045, 0.00044, 0.00046, 0.00044, 0.00045, 0.00059, 0.00045, 0.00045, 0.00045, 0.00045, 0.00045, 0.00044, 0.00045, 0.00046, 0.00046, 0.00052, 0.00046, 0.00045, 0.00044, 0.00044, 0.00045, 0.00043, 0.00046, 0.00045, 0.00045, 0.00046, 0.00049, 0.00046, 0.00045, 0.00046, 0.00049, 0.00045, 0.00043, 0.00044, 0.00044, 0.00046, 0.00056, 0.00044]}, "optimizer-copy-main-to-model-params-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.00074, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00057, 0.00047, 0.00067, 0.00046, 0.0005, 0.00046, 0.00046, 0.00046, 0.00049, 0.00047, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00064, 0.00046, 0.00049, 0.00047, 0.00047, 0.00053, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.0005, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00046, 0.00072, 0.00048, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00053, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00049, 0.00047, 0.00047, 0.00046, 0.00047, 0.0005, 0.00047, 0.00047, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00048, 0.00048, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.0005, 0.00046, 0.00046, 0.00047, 0.00046, 0.00066, 0.00046, 0.00046, 0.00047, 0.00046, 0.00048, 0.00048, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00047, 0.00047, 0.0007, 0.00046, 0.00047, 0.00046, 0.00047, 0.0005, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00047, 0.00047, 0.00048, 0.00047, 0.00047, 0.00048, 0.00047, 0.00049, 0.00046, 0.00047, 0.00046, 0.00047, 0.00049, 0.00046, 0.00046, 0.00047, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00057, 0.00046, 0.00046, 0.00046, 0.00072, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00051, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00048, 0.00047, 0.00047, 0.00047, 0.00047, 0.00047, 0.0005, 0.00047, 0.00047, 0.00048, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00046, 0.00047, 0.00069, 0.00061, 0.00061, 0.00062, 0.00063, 0.00063, 0.00061, 0.00062, 0.00062, 0.00062, 0.00061, 0.00062, 0.00062, 0.00063, 0.00062, 0.00062, 0.00074, 0.00062, 0.00061, 0.00062, 0.00062, 0.00064, 0.00062, 0.00061, 0.00062, 0.00062, 0.00061, 0.00062, 0.00063, 0.00062, 0.00062, 0.00062, 0.00062, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00049, 0.00047, 0.00049, 0.00046, 0.00049, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00072, 0.00049, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00064, 0.00048, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00051, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.0005, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00047, 0.00047, 0.00047, 0.00048, 0.00046, 0.00046, 0.00046, 0.00046, 0.00046, 0.00048, 0.00047, 0.00047, 0.00047, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.0007, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00047, 0.00046, 0.00051, 0.00048, 0.00047, 0.00046, 0.00047, 0.00046, 0.00047, 0.00047, 0.00046, 0.00046, 0.00047, 0.00047, 0.00048, 0.00046, 0.00047, 0.0005, 0.00046, 0.00047, 0.00046, 0.00046, 0.00046, 0.00065, 0.00047]}, "optimizer-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [6.53084, 0.00464, 0.00458, 0.0046, 0.00463, 0.00462, 0.00461, 0.0046, 0.00462, 0.00466, 0.00468, 0.00464, 0.00464, 0.00464, 0.00466, 0.00465, 0.00461, 0.00462, 0.0046, 0.00459, 0.00462, 0.00459, 0.0046, 0.00474, 0.0046, 0.0046, 0.00459, 0.00461, 0.00533, 0.00461, 0.00562, 0.00464, 0.00716, 0.00471, 0.00463, 0.00461, 0.00461, 0.00462, 0.00462, 0.00465, 0.00464, 0.00461, 0.00459, 0.00463, 0.00464, 0.0046, 0.00459, 0.00494, 0.00461, 0.00464, 0.00472, 0.00463, 0.00467, 0.00463, 0.00461, 0.00461, 0.00461, 0.00459, 0.00465, 0.00478, 0.00462, 0.00464, 0.0046, 0.00464, 0.00461, 0.00462, 0.00484, 0.00467, 0.00469, 0.00458, 0.00458, 0.00458, 0.00459, 0.00459, 0.00474, 0.00455, 0.00464, 0.00458, 0.00457, 0.0046, 0.00458, 0.0046, 0.0047, 0.00458, 0.00459, 0.00468, 0.00458, 0.00456, 0.00459, 0.00458, 0.00454, 0.00457, 0.00454, 0.00535, 0.00469, 0.00459, 0.00457, 0.0046, 0.00459, 0.00459, 0.00458, 0.0046, 0.00456, 0.00459, 0.00551, 0.00461, 0.00463, 0.00451, 0.00459, 0.00451, 0.00449, 0.00453, 0.00459, 0.00458, 0.00454, 0.00456, 0.00458, 0.00462, 0.00451, 0.00457, 0.00461, 0.0046, 0.00497, 0.00461, 0.00455, 0.00458, 0.00469, 0.00472, 0.0046, 0.00459, 0.00459, 0.0046, 0.00457, 0.0046, 0.00462, 0.00461, 0.00458, 0.00464, 0.00459, 0.0046, 0.00465, 0.00469, 0.00462, 0.00463, 0.00463, 0.00463, 0.00518, 0.00462, 0.00478, 0.00458, 0.00463, 0.00462, 0.00466, 0.00465, 0.00463, 0.0048, 0.00458, 0.00458, 0.00458, 0.00461, 0.00458, 0.00461, 0.00505, 0.00457, 0.00461, 0.00456, 0.00461, 0.00463, 0.00467, 0.00457, 0.0046, 0.00454, 0.00459, 0.00462, 0.00461, 0.00459, 0.00465, 0.00457, 0.0046, 0.00457, 0.00459, 0.00461, 0.00563, 0.00466, 0.00459, 0.00456, 0.00458, 0.00457, 0.00457, 0.00462, 0.00476, 0.00461, 0.00459, 0.00458, 0.00478, 0.00458, 0.00498, 0.00465, 0.00458, 0.00462, 0.00441, 0.00438, 0.00432, 0.00434, 0.00433, 0.00431, 0.00434, 0.00431, 0.00433, 0.00433, 0.00454, 0.00435, 0.00437, 0.00435, 0.00489, 0.00436, 0.00436, 0.00435, 0.00438, 0.00436, 0.00432, 0.00433, 0.00433, 0.00437, 0.00441, 0.00434, 0.00434, 0.00432, 0.00434, 0.0044, 0.00432, 0.0044, 0.00432, 0.00431, 0.00433, 0.00442, 0.00438, 0.00454, 0.00434, 0.00437, 0.00523, 0.00436, 0.00437, 0.00435, 0.00437, 0.00436, 0.00435, 0.00441, 0.00694, 0.00622, 0.00624, 0.00622, 0.00629, 0.00622, 0.0062, 0.0062, 0.00622, 0.00645, 0.00629, 0.00622, 0.00619, 0.00626, 0.0062, 0.00622, 0.00688, 0.00622, 0.00622, 0.00623, 0.00625, 0.00629, 0.00647, 0.00622, 0.00622, 0.00625, 0.00625, 0.00629, 0.00622, 0.0062, 0.00624, 0.00622, 0.00626, 0.00434, 0.00431, 0.00435, 0.0043, 0.00431, 0.00428, 0.00427, 0.00431, 0.00429, 0.00435, 0.00428, 0.00431, 0.00431, 0.00433, 0.00435, 0.00433, 0.00428, 0.00432, 0.00428, 0.00432, 0.00427, 0.00434, 0.0043, 0.00485, 0.00439, 0.00433, 0.00428, 0.0043, 0.00428, 0.00429, 0.00428, 0.0043, 0.00432, 0.00427, 0.00475, 0.00433, 0.0043, 0.00434, 0.00432, 0.00436, 0.00428, 0.00429, 0.00429, 0.00429, 0.00433, 0.0043, 0.00428, 0.00433, 0.0043, 0.00433, 0.00427, 0.00427, 0.00439, 0.00443, 0.00428, 0.00431, 0.00426, 0.00429, 0.0043, 0.00426, 0.00441, 0.00428, 0.0043, 0.00436, 0.00429, 0.00431, 0.00428, 0.00462, 0.00436, 0.00436, 0.00431, 0.00439, 0.00429, 0.00433, 0.00433, 0.00433, 0.00453, 0.00436, 0.00436, 0.00432, 0.00435, 0.00441, 0.00431, 0.00437, 0.00436, 0.00437, 0.00495, 0.00431, 0.00434, 0.00433, 0.00433, 0.00438, 0.00429, 0.00433, 0.00433, 0.00431, 0.0054, 0.00436, 0.00437, 0.00433, 0.0043, 0.0044, 0.0043, 0.00436, 0.00431, 0.00431, 0.00435, 0.00472, 0.00451, 0.00436, 0.00433, 0.0047, 0.00432, 0.00427, 0.00432, 0.00431, 0.0044, 0.00518, 0.00433]}, "learning-rate": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "learning-rate vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 1e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 2e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 3e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 4e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 5e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 6e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 7e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 8e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05, 9e-05]}, "batch-size": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "batch-size vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0, 128.0]}, "lm loss": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.89904, 10.90777, 10.89232, 10.83544, 10.6834, 10.65974, 10.44873, 10.16308, 9.95831, 9.85932, 9.60254, 9.85446, 9.88893, 9.63287, 9.79405, 9.51078, 9.46463, 9.65471, 9.39306, 9.33895, 9.24972, 9.15413, 9.17988, 9.0065, 9.19899, 9.06474, 9.16249, 9.16631, 9.30043, 8.98957, 8.93842, 9.05744, 9.05222, 8.66356, 8.72626, 8.7667, 8.70006, 8.74817, 8.67179, 8.78274, 8.67795, 8.86767, 8.84929, 8.51536, 8.40624, 8.45093, 8.51004, 8.40653, 8.45216, 8.6026, 8.38502, 8.21394, 8.24297, 8.23879, 8.28518, 7.93123, 8.10705, 7.90575, 8.25948, 8.24016, 8.01415, 7.97894, 7.93174, 7.74864, 7.74918, 7.65293, 7.52384, 7.91349, 7.70509, 7.46214, 7.74596, 7.77384, 7.5447, 7.30561, 7.45871, 7.34545, 7.46856, 7.23017, 7.64088, 7.27983, 7.34981, 7.21134, 7.21081, 7.42102, 7.17384, 7.28052, 6.99786, 7.00152, 7.03624, 7.13136, 6.82298, 6.98762, 7.08699, 6.99714, 6.87231, 6.75444, 6.98392, 7.05773, 6.69999, 6.57801, 6.72248, 6.73865, 6.73005, 6.73698, 6.65374, 6.40729, 6.6365, 6.61972, 6.44423, 6.62637, 6.74067, 6.60551, 6.72345, 6.68935, 6.62052, 6.50773, 6.59703, 6.40181, 6.66219, 6.24576, 6.24815, 6.29992, 6.38652, 6.34284, 6.44395, 6.2868, 6.33137, 6.23064, 6.19419, 6.38932, 6.31955, 6.31115, 6.15595, 6.14904, 6.23012, 6.37609, 6.19108, 6.14016, 6.17443, 6.108, 6.05677, 6.07051, 6.2515, 6.40359, 6.25653, 6.30179, 6.09464, 6.1786, 6.00393, 6.03024, 5.95456, 6.25097, 6.18949, 5.96652, 5.78509, 6.12471, 5.85239, 6.09954, 5.78907, 6.1634, 6.14662, 6.08899, 5.93324, 6.11629, 5.94863, 6.19744, 5.89699, 5.79464, 5.78508, 5.6887, 6.01484, 5.99513, 6.06793, 5.88964, 6.04218, 5.96664, 5.9946, 5.98873, 5.94909, 5.83777, 5.94965, 5.62073, 5.70203, 5.88937, 5.84442, 5.86415, 5.75977, 5.83426, 5.72464, 5.56351, 5.71986, 5.62642, 5.83426, 5.60742, 5.71258, 5.70976, 5.8987, 5.64295, 5.85277, 5.73889, 5.87053, 5.32966, 5.89533, 5.87205, 5.85426, 5.41037, 5.40663, 5.62114, 5.59572, 5.48482, 5.57586, 5.67197, 5.4726, 5.74298, 5.50672, 5.5935, 5.61776, 5.6179, 5.51203, 5.61413, 5.67291, 5.68327, 5.58724, 5.66009, 5.37678, 5.68099, 5.62359, 5.42053, 5.57867, 5.62946, 5.54954, 5.33822, 5.53445, 5.48149, 5.47842, 5.37511, 5.5464, 5.60351, 5.38706, 5.51715, 5.48729, 5.33094, 5.50178, 5.40732, 5.44712, 5.31548, 5.06617, 5.47969, 5.56831, 5.7133, 5.41401, 5.59841, 5.63558, 5.2322, 5.27319, 5.38792, 5.39306, 5.32904, 5.49509, 5.17834, 5.29764, 5.24393, 5.37614, 5.25456, 5.44258, 5.54017, 5.31017, 5.43225, 5.33341, 5.07298, 5.31187, 5.2557, 5.30514, 5.10844, 5.27459, 5.26496, 5.47616, 5.16669, 5.26555, 5.21176, 5.355, 4.98377, 4.91178, 5.33096, 5.38935, 5.23414, 5.31329, 5.10388, 5.16417, 5.26356, 5.06801, 5.27045, 5.07377, 5.34602, 5.24563, 5.15001, 5.24094, 5.04069, 5.31488, 5.04958, 5.02979, 5.13788, 5.11434, 5.26734, 5.14852, 5.27369, 5.08851, 5.09324, 5.24624, 5.32324, 5.25443, 5.19052, 5.14435, 5.29055, 4.94885, 5.20441, 5.0907, 5.29874, 5.17267, 5.18858, 5.11677, 4.98159, 4.99122, 5.22123, 5.30764, 5.10222, 5.0544, 4.91358, 5.12177, 5.11614, 4.92915, 5.33612, 5.01913, 5.10051, 5.16573, 4.99929, 5.06049, 5.06814, 4.99437, 5.07642, 5.16464, 4.98109, 5.1825, 4.92945, 4.92916, 5.06868, 4.99902, 4.90979, 4.77687, 4.94499, 5.11671, 5.01541, 5.02126, 5.32954, 4.95713, 4.99895, 5.05055, 4.81011, 4.73872, 5.00091, 5.04398, 4.87805, 4.95233, 5.04347, 5.02539, 4.82104, 4.90025, 4.90912, 4.83747, 4.75039, 5.01482, 4.74829, 5.21037, 4.79047, 5.00245, 4.74175, 4.79189, 4.82107, 4.65381, 4.66051, 4.84616, 4.81073, 4.8078, 4.92405, 4.88723, 4.93597, 4.77468, 4.88361, 4.74125, 4.92209, 4.96252, 4.87874, 4.71289, 4.79114, 4.90017, 4.7175, 4.87202, 4.69846, 4.70626, 4.65256]}, "lm loss vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [10.89904, 10.90777, 10.89232, 10.83544, 10.6834, 10.65974, 10.44873, 10.16308, 9.95831, 9.85932, 9.60254, 9.85446, 9.88893, 9.63287, 9.79405, 9.51078, 9.46463, 9.65471, 9.39306, 9.33895, 9.24972, 9.15413, 9.17988, 9.0065, 9.19899, 9.06474, 9.16249, 9.16631, 9.30043, 8.98957, 8.93842, 9.05744, 9.05222, 8.66356, 8.72626, 8.7667, 8.70006, 8.74817, 8.67179, 8.78274, 8.67795, 8.86767, 8.84929, 8.51536, 8.40624, 8.45093, 8.51004, 8.40653, 8.45216, 8.6026, 8.38502, 8.21394, 8.24297, 8.23879, 8.28518, 7.93123, 8.10705, 7.90575, 8.25948, 8.24016, 8.01415, 7.97894, 7.93174, 7.74864, 7.74918, 7.65293, 7.52384, 7.91349, 7.70509, 7.46214, 7.74596, 7.77384, 7.5447, 7.30561, 7.45871, 7.34545, 7.46856, 7.23017, 7.64088, 7.27983, 7.34981, 7.21134, 7.21081, 7.42102, 7.17384, 7.28052, 6.99786, 7.00152, 7.03624, 7.13136, 6.82298, 6.98762, 7.08699, 6.99714, 6.87231, 6.75444, 6.98392, 7.05773, 6.69999, 6.57801, 6.72248, 6.73865, 6.73005, 6.73698, 6.65374, 6.40729, 6.6365, 6.61972, 6.44423, 6.62637, 6.74067, 6.60551, 6.72345, 6.68935, 6.62052, 6.50773, 6.59703, 6.40181, 6.66219, 6.24576, 6.24815, 6.29992, 6.38652, 6.34284, 6.44395, 6.2868, 6.33137, 6.23064, 6.19419, 6.38932, 6.31955, 6.31115, 6.15595, 6.14904, 6.23012, 6.37609, 6.19108, 6.14016, 6.17443, 6.108, 6.05677, 6.07051, 6.2515, 6.40359, 6.25653, 6.30179, 6.09464, 6.1786, 6.00393, 6.03024, 5.95456, 6.25097, 6.18949, 5.96652, 5.78509, 6.12471, 5.85239, 6.09954, 5.78907, 6.1634, 6.14662, 6.08899, 5.93324, 6.11629, 5.94863, 6.19744, 5.89699, 5.79464, 5.78508, 5.6887, 6.01484, 5.99513, 6.06793, 5.88964, 6.04218, 5.96664, 5.9946, 5.98873, 5.94909, 5.83777, 5.94965, 5.62073, 5.70203, 5.88937, 5.84442, 5.86415, 5.75977, 5.83426, 5.72464, 5.56351, 5.71986, 5.62642, 5.83426, 5.60742, 5.71258, 5.70976, 5.8987, 5.64295, 5.85277, 5.73889, 5.87053, 5.32966, 5.89533, 5.87205, 5.85426, 5.41037, 5.40663, 5.62114, 5.59572, 5.48482, 5.57586, 5.67197, 5.4726, 5.74298, 5.50672, 5.5935, 5.61776, 5.6179, 5.51203, 5.61413, 5.67291, 5.68327, 5.58724, 5.66009, 5.37678, 5.68099, 5.62359, 5.42053, 5.57867, 5.62946, 5.54954, 5.33822, 5.53445, 5.48149, 5.47842, 5.37511, 5.5464, 5.60351, 5.38706, 5.51715, 5.48729, 5.33094, 5.50178, 5.40732, 5.44712, 5.31548, 5.06617, 5.47969, 5.56831, 5.7133, 5.41401, 5.59841, 5.63558, 5.2322, 5.27319, 5.38792, 5.39306, 5.32904, 5.49509, 5.17834, 5.29764, 5.24393, 5.37614, 5.25456, 5.44258, 5.54017, 5.31017, 5.43225, 5.33341, 5.07298, 5.31187, 5.2557, 5.30514, 5.10844, 5.27459, 5.26496, 5.47616, 5.16669, 5.26555, 5.21176, 5.355, 4.98377, 4.91178, 5.33096, 5.38935, 5.23414, 5.31329, 5.10388, 5.16417, 5.26356, 5.06801, 5.27045, 5.07377, 5.34602, 5.24563, 5.15001, 5.24094, 5.04069, 5.31488, 5.04958, 5.02979, 5.13788, 5.11434, 5.26734, 5.14852, 5.27369, 5.08851, 5.09324, 5.24624, 5.32324, 5.25443, 5.19052, 5.14435, 5.29055, 4.94885, 5.20441, 5.0907, 5.29874, 5.17267, 5.18858, 5.11677, 4.98159, 4.99122, 5.22123, 5.30764, 5.10222, 5.0544, 4.91358, 5.12177, 5.11614, 4.92915, 5.33612, 5.01913, 5.10051, 5.16573, 4.99929, 5.06049, 5.06814, 4.99437, 5.07642, 5.16464, 4.98109, 5.1825, 4.92945, 4.92916, 5.06868, 4.99902, 4.90979, 4.77687, 4.94499, 5.11671, 5.01541, 5.02126, 5.32954, 4.95713, 4.99895, 5.05055, 4.81011, 4.73872, 5.00091, 5.04398, 4.87805, 4.95233, 5.04347, 5.02539, 4.82104, 4.90025, 4.90912, 4.83747, 4.75039, 5.01482, 4.74829, 5.21037, 4.79047, 5.00245, 4.74175, 4.79189, 4.82107, 4.65381, 4.66051, 4.84616, 4.81073, 4.8078, 4.92405, 4.88723, 4.93597, 4.77468, 4.88361, 4.74125, 4.92209, 4.96252, 4.87874, 4.71289, 4.79114, 4.90017, 4.7175, 4.87202, 4.69846, 4.70626, 4.65256]}, "loss-scale": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "loss-scale vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]}, "grad-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.85752, 13.16701, 13.66167, 12.68371, 12.08638, 9.51321, 6.94209, 7.08694, 6.10814, 4.68821, 4.2751, 2.87984, 2.44435, 2.3806, 2.05602, 2.21803, 2.17031, 1.89335, 2.22351, 2.07816, 2.13217, 2.16577, 2.02595, 2.23917, 2.00742, 2.14445, 1.91002, 1.89231, 1.93089, 2.06379, 2.16765, 2.23679, 1.89668, 2.34753, 2.35194, 2.16267, 2.15162, 1.83098, 2.05276, 1.74395, 2.36831, 1.97031, 1.80751, 1.87923, 1.94701, 1.80892, 1.71885, 1.77109, 1.75698, 1.55174, 1.76422, 1.75578, 1.7467, 1.926, 1.6754, 1.89063, 1.76173, 1.82379, 1.52589, 1.48723, 1.63648, 1.49118, 1.79292, 1.82033, 1.59591, 1.62383, 1.63898, 1.62368, 1.43237, 1.62305, 1.35226, 1.37441, 1.77832, 1.4053, 1.36387, 1.43489, 1.33927, 1.41507, 1.32726, 1.26584, 1.3881, 1.23171, 1.40194, 1.20354, 1.1842, 1.32033, 1.50387, 1.25756, 1.20187, 1.05786, 1.15737, 1.22128, 1.02487, 1.08879, 0.98695, 1.28999, 0.98417, 1.58629, 1.03703, 1.06213, 1.55961, 1.47669, 0.90784, 1.45527, 1.29065, 1.13286, 1.14779, 0.95484, 1.09964, 0.89588, 0.84205, 0.91582, 1.04481, 1.01608, 1.02993, 1.12143, 1.08948, 1.31986, 0.92092, 1.1799, 1.09173, 1.10393, 1.19122, 1.03752, 1.03062, 1.19126, 1.02231, 1.0955, 1.05064, 1.06655, 1.1517, 1.11568, 1.37446, 1.21005, 1.53165, 1.24599, 1.03436, 1.56617, 1.39613, 1.20613, 1.59751, 1.76157, 1.17134, 1.06152, 1.22514, 1.97917, 1.11879, 1.62597, 1.18846, 0.95412, 1.17247, 1.50913, 1.42049, 1.32267, 1.02991, 1.60853, 1.51052, 1.23861, 1.4438, 1.81637, 1.43133, 1.52934, 1.66869, 1.18507, 1.38099, 1.44638, 1.56369, 1.1851, 1.63779, 1.22939, 1.13585, 0.93198, 1.58024, 1.61619, 1.48199, 1.39642, 1.72479, 1.20982, 1.33257, 1.14605, 1.14908, 1.46659, 1.41611, 1.64334, 1.40953, 1.89405, 1.62101, 1.55, 1.25036, 1.73578, 1.20849, 1.16164, 2.00175, 1.79359, 1.54068, 1.27095, 1.51292, 1.45211, 1.55181, 1.38317, 1.19552, 1.41924, 1.0843, 1.11099, 1.49128, 1.31175, 1.31568, 1.31643, 1.38944, 1.83714, 1.51633, 1.66291, 1.32027, 1.40224, 1.23381, 1.24726, 1.17329, 1.41173, 1.41298, 1.21975, 1.40395, 1.29766, 1.647, 1.77185, 1.70549, 1.66243, 1.35144, 1.53811, 1.34558, 1.49398, 1.11503, 1.29778, 1.74207, 1.44213, 1.53886, 1.63632, 1.20482, 1.57111, 1.4054, 1.21748, 1.63569, 1.23136, 1.58159, 1.59579, 1.48012, 1.5323, 1.55081, 1.4194, 1.57228, 1.48387, 1.38849, 1.27392, 1.46178, 1.25824, 1.36062, 1.39751, 1.30771, 1.33147, 1.56583, 1.32709, 1.3646, 1.55907, 1.61002, 1.45173, 1.42035, 2.16284, 1.75737, 1.67782, 1.31786, 1.45228, 1.59778, 1.56015, 1.4983, 1.23696, 1.35268, 1.40317, 1.37404, 1.67666, 1.49364, 1.47162, 1.50218, 1.40879, 1.26151, 1.53009, 1.2357, 1.52653, 1.16029, 1.37287, 1.45359, 1.43811, 1.48164, 1.84101, 1.47755, 1.57834, 1.61834, 1.37842, 1.4784, 1.5761, 1.25832, 1.22282, 1.47102, 1.22564, 1.24267, 1.4204, 1.52394, 1.4913, 1.42263, 1.42192, 1.14735, 1.34499, 1.41439, 1.29824, 1.69085, 1.44146, 1.55667, 1.25423, 1.36428, 1.18219, 1.19336, 1.33449, 1.6401, 1.40383, 1.31292, 1.52789, 1.3215, 1.5794, 1.52614, 1.22037, 1.55665, 1.33214, 1.42978, 1.54699, 1.14418, 1.6388, 1.34807, 1.3749, 1.28337, 1.39417, 1.59994, 1.36359, 1.36119, 1.19917, 1.33658, 1.27596, 1.44996, 1.61368, 1.41282, 1.45175, 1.23245, 1.34616, 1.42121, 1.22977, 1.59453, 1.46628, 1.2612, 1.66869, 1.34891, 1.38326, 1.54549, 1.62587, 1.50361, 1.33282, 1.30675, 1.24628, 1.22264, 1.39221, 1.62236, 1.59048, 1.51538, 1.71681, 1.34251, 1.22656, 1.61992, 1.40775, 1.39241, 1.37966, 1.26457, 1.31626, 1.23459, 1.33073, 1.25512, 1.32646, 1.32216, 1.2607, 1.26972, 1.41721, 1.4656, 1.22975, 1.33206, 1.36899, 1.3651, 1.49566, 1.54131, 1.24469, 1.32355, 1.39775, 1.35713, 1.23875, 1.37455, 1.14642]}, "grad-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [13.85752, 13.16701, 13.66167, 12.68371, 12.08638, 9.51321, 6.94209, 7.08694, 6.10814, 4.68821, 4.2751, 2.87984, 2.44435, 2.3806, 2.05602, 2.21803, 2.17031, 1.89335, 2.22351, 2.07816, 2.13217, 2.16577, 2.02595, 2.23917, 2.00742, 2.14445, 1.91002, 1.89231, 1.93089, 2.06379, 2.16765, 2.23679, 1.89668, 2.34753, 2.35194, 2.16267, 2.15162, 1.83098, 2.05276, 1.74395, 2.36831, 1.97031, 1.80751, 1.87923, 1.94701, 1.80892, 1.71885, 1.77109, 1.75698, 1.55174, 1.76422, 1.75578, 1.7467, 1.926, 1.6754, 1.89063, 1.76173, 1.82379, 1.52589, 1.48723, 1.63648, 1.49118, 1.79292, 1.82033, 1.59591, 1.62383, 1.63898, 1.62368, 1.43237, 1.62305, 1.35226, 1.37441, 1.77832, 1.4053, 1.36387, 1.43489, 1.33927, 1.41507, 1.32726, 1.26584, 1.3881, 1.23171, 1.40194, 1.20354, 1.1842, 1.32033, 1.50387, 1.25756, 1.20187, 1.05786, 1.15737, 1.22128, 1.02487, 1.08879, 0.98695, 1.28999, 0.98417, 1.58629, 1.03703, 1.06213, 1.55961, 1.47669, 0.90784, 1.45527, 1.29065, 1.13286, 1.14779, 0.95484, 1.09964, 0.89588, 0.84205, 0.91582, 1.04481, 1.01608, 1.02993, 1.12143, 1.08948, 1.31986, 0.92092, 1.1799, 1.09173, 1.10393, 1.19122, 1.03752, 1.03062, 1.19126, 1.02231, 1.0955, 1.05064, 1.06655, 1.1517, 1.11568, 1.37446, 1.21005, 1.53165, 1.24599, 1.03436, 1.56617, 1.39613, 1.20613, 1.59751, 1.76157, 1.17134, 1.06152, 1.22514, 1.97917, 1.11879, 1.62597, 1.18846, 0.95412, 1.17247, 1.50913, 1.42049, 1.32267, 1.02991, 1.60853, 1.51052, 1.23861, 1.4438, 1.81637, 1.43133, 1.52934, 1.66869, 1.18507, 1.38099, 1.44638, 1.56369, 1.1851, 1.63779, 1.22939, 1.13585, 0.93198, 1.58024, 1.61619, 1.48199, 1.39642, 1.72479, 1.20982, 1.33257, 1.14605, 1.14908, 1.46659, 1.41611, 1.64334, 1.40953, 1.89405, 1.62101, 1.55, 1.25036, 1.73578, 1.20849, 1.16164, 2.00175, 1.79359, 1.54068, 1.27095, 1.51292, 1.45211, 1.55181, 1.38317, 1.19552, 1.41924, 1.0843, 1.11099, 1.49128, 1.31175, 1.31568, 1.31643, 1.38944, 1.83714, 1.51633, 1.66291, 1.32027, 1.40224, 1.23381, 1.24726, 1.17329, 1.41173, 1.41298, 1.21975, 1.40395, 1.29766, 1.647, 1.77185, 1.70549, 1.66243, 1.35144, 1.53811, 1.34558, 1.49398, 1.11503, 1.29778, 1.74207, 1.44213, 1.53886, 1.63632, 1.20482, 1.57111, 1.4054, 1.21748, 1.63569, 1.23136, 1.58159, 1.59579, 1.48012, 1.5323, 1.55081, 1.4194, 1.57228, 1.48387, 1.38849, 1.27392, 1.46178, 1.25824, 1.36062, 1.39751, 1.30771, 1.33147, 1.56583, 1.32709, 1.3646, 1.55907, 1.61002, 1.45173, 1.42035, 2.16284, 1.75737, 1.67782, 1.31786, 1.45228, 1.59778, 1.56015, 1.4983, 1.23696, 1.35268, 1.40317, 1.37404, 1.67666, 1.49364, 1.47162, 1.50218, 1.40879, 1.26151, 1.53009, 1.2357, 1.52653, 1.16029, 1.37287, 1.45359, 1.43811, 1.48164, 1.84101, 1.47755, 1.57834, 1.61834, 1.37842, 1.4784, 1.5761, 1.25832, 1.22282, 1.47102, 1.22564, 1.24267, 1.4204, 1.52394, 1.4913, 1.42263, 1.42192, 1.14735, 1.34499, 1.41439, 1.29824, 1.69085, 1.44146, 1.55667, 1.25423, 1.36428, 1.18219, 1.19336, 1.33449, 1.6401, 1.40383, 1.31292, 1.52789, 1.3215, 1.5794, 1.52614, 1.22037, 1.55665, 1.33214, 1.42978, 1.54699, 1.14418, 1.6388, 1.34807, 1.3749, 1.28337, 1.39417, 1.59994, 1.36359, 1.36119, 1.19917, 1.33658, 1.27596, 1.44996, 1.61368, 1.41282, 1.45175, 1.23245, 1.34616, 1.42121, 1.22977, 1.59453, 1.46628, 1.2612, 1.66869, 1.34891, 1.38326, 1.54549, 1.62587, 1.50361, 1.33282, 1.30675, 1.24628, 1.22264, 1.39221, 1.62236, 1.59048, 1.51538, 1.71681, 1.34251, 1.22656, 1.61992, 1.40775, 1.39241, 1.37966, 1.26457, 1.31626, 1.23459, 1.33073, 1.25512, 1.32646, 1.32216, 1.2607, 1.26972, 1.41721, 1.4656, 1.22975, 1.33206, 1.36899, 1.3651, 1.49566, 1.54131, 1.24469, 1.32355, 1.39775, 1.35713, 1.23875, 1.37455, 1.14642]}, "num-zeros": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [58.0, 87.0, 81.0, 84.0, 84.0, 90.0, 104.0, 124.0, 102.0, 132.0, 129.0, 152.0, 143.0, 181.0, 202.0, 161.0, 161.0, 177.0, 184.0, 189.0, 151.0, 167.0, 183.0, 182.0, 186.0, 154.0, 178.0, 163.0, 167.0, 148.0, 145.0, 138.0, 187.0, 168.0, 140.0, 142.0, 167.0, 204.0, 169.0, 203.0, 148.0, 155.0, 141.0, 200.0, 190.0, 169.0, 187.0, 196.0, 175.0, 229.0, 207.0, 188.0, 199.0, 157.0, 186.0, 178.0, 154.0, 138.0, 248.0, 232.0, 174.0, 186.0, 188.0, 193.0, 201.0, 239.0, 207.0, 166.0, 208.0, 203.0, 208.0, 254.0, 168.0, 251.0, 210.0, 201.0, 239.0, 211.0, 241.0, 211.0, 204.0, 215.0, 193.0, 225.0, 213.0, 184.0, 182.0, 191.0, 206.0, 206.0, 188.0, 218.0, 214.0, 205.0, 203.0, 166.0, 206.0, 174.0, 195.0, 174.0, 140.0, 154.0, 176.0, 165.0, 129.0, 148.0, 168.0, 157.0, 137.0, 180.0, 175.0, 163.0, 175.0, 145.0, 138.0, 134.0, 159.0, 128.0, 173.0, 161.0, 151.0, 113.0, 133.0, 129.0, 177.0, 125.0, 153.0, 137.0, 120.0, 142.0, 148.0, 143.0, 100.0, 113.0, 106.0, 124.0, 129.0, 93.0, 119.0, 125.0, 107.0, 107.0, 141.0, 141.0, 122.0, 91.0, 142.0, 120.0, 101.0, 141.0, 130.0, 112.0, 107.0, 110.0, 132.0, 105.0, 102.0, 116.0, 115.0, 122.0, 96.0, 122.0, 87.0, 104.0, 112.0, 91.0, 110.0, 107.0, 101.0, 103.0, 107.0, 117.0, 83.0, 102.0, 105.0, 133.0, 96.0, 115.0, 93.0, 128.0, 129.0, 113.0, 112.0, 104.0, 104.0, 90.0, 85.0, 92.0, 96.0, 79.0, 140.0, 112.0, 103.0, 85.0, 96.0, 103.0, 104.0, 90.0, 109.0, 115.0, 113.0, 82.0, 123.0, 128.0, 86.0, 113.0, 103.0, 100.0, 129.0, 90.0, 96.0, 92.0, 106.0, 106.0, 113.0, 127.0, 112.0, 118.0, 96.0, 106.0, 114.0, 93.0, 85.0, 74.0, 105.0, 113.0, 97.0, 113.0, 107.0, 97.0, 109.0, 87.0, 89.0, 108.0, 106.0, 87.0, 120.0, 115.0, 109.0, 111.0, 100.0, 114.0, 102.0, 106.0, 94.0, 106.0, 77.0, 124.0, 112.0, 102.0, 104.0, 111.0, 109.0, 125.0, 114.0, 109.0, 120.0, 120.0, 103.0, 107.0, 86.0, 111.0, 95.0, 102.0, 108.0, 78.0, 100.0, 90.0, 107.0, 101.0, 104.0, 119.0, 100.0, 113.0, 110.0, 113.0, 90.0, 101.0, 107.0, 106.0, 111.0, 88.0, 125.0, 93.0, 106.0, 103.0, 116.0, 127.0, 100.0, 84.0, 102.0, 97.0, 97.0, 94.0, 120.0, 109.0, 110.0, 98.0, 97.0, 113.0, 108.0, 106.0, 143.0, 104.0, 111.0, 106.0, 103.0, 99.0, 110.0, 106.0, 130.0, 121.0, 112.0, 103.0, 101.0, 97.0, 115.0, 127.0, 117.0, 116.0, 109.0, 101.0, 129.0, 101.0, 99.0, 112.0, 91.0, 113.0, 104.0, 122.0, 91.0, 120.0, 124.0, 89.0, 106.0, 106.0, 119.0, 101.0, 98.0, 102.0, 129.0, 107.0, 116.0, 126.0, 127.0, 112.0, 86.0, 106.0, 136.0, 135.0, 107.0, 93.0, 102.0, 118.0, 117.0, 104.0, 123.0, 99.0, 114.0, 92.0, 128.0, 92.0, 107.0, 92.0, 124.0, 106.0, 101.0, 112.0, 106.0, 99.0, 107.0, 110.0, 97.0, 108.0, 117.0, 119.0, 102.0, 116.0, 116.0, 118.0, 108.0, 130.0, 116.0, 118.0, 122.0, 105.0, 104.0, 126.0, 123.0, 118.0, 124.0, 126.0, 97.0, 123.0, 133.0, 101.0, 117.0, 114.0, 120.0, 139.0]}, "num-zeros vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [58.0, 87.0, 81.0, 84.0, 84.0, 90.0, 104.0, 124.0, 102.0, 132.0, 129.0, 152.0, 143.0, 181.0, 202.0, 161.0, 161.0, 177.0, 184.0, 189.0, 151.0, 167.0, 183.0, 182.0, 186.0, 154.0, 178.0, 163.0, 167.0, 148.0, 145.0, 138.0, 187.0, 168.0, 140.0, 142.0, 167.0, 204.0, 169.0, 203.0, 148.0, 155.0, 141.0, 200.0, 190.0, 169.0, 187.0, 196.0, 175.0, 229.0, 207.0, 188.0, 199.0, 157.0, 186.0, 178.0, 154.0, 138.0, 248.0, 232.0, 174.0, 186.0, 188.0, 193.0, 201.0, 239.0, 207.0, 166.0, 208.0, 203.0, 208.0, 254.0, 168.0, 251.0, 210.0, 201.0, 239.0, 211.0, 241.0, 211.0, 204.0, 215.0, 193.0, 225.0, 213.0, 184.0, 182.0, 191.0, 206.0, 206.0, 188.0, 218.0, 214.0, 205.0, 203.0, 166.0, 206.0, 174.0, 195.0, 174.0, 140.0, 154.0, 176.0, 165.0, 129.0, 148.0, 168.0, 157.0, 137.0, 180.0, 175.0, 163.0, 175.0, 145.0, 138.0, 134.0, 159.0, 128.0, 173.0, 161.0, 151.0, 113.0, 133.0, 129.0, 177.0, 125.0, 153.0, 137.0, 120.0, 142.0, 148.0, 143.0, 100.0, 113.0, 106.0, 124.0, 129.0, 93.0, 119.0, 125.0, 107.0, 107.0, 141.0, 141.0, 122.0, 91.0, 142.0, 120.0, 101.0, 141.0, 130.0, 112.0, 107.0, 110.0, 132.0, 105.0, 102.0, 116.0, 115.0, 122.0, 96.0, 122.0, 87.0, 104.0, 112.0, 91.0, 110.0, 107.0, 101.0, 103.0, 107.0, 117.0, 83.0, 102.0, 105.0, 133.0, 96.0, 115.0, 93.0, 128.0, 129.0, 113.0, 112.0, 104.0, 104.0, 90.0, 85.0, 92.0, 96.0, 79.0, 140.0, 112.0, 103.0, 85.0, 96.0, 103.0, 104.0, 90.0, 109.0, 115.0, 113.0, 82.0, 123.0, 128.0, 86.0, 113.0, 103.0, 100.0, 129.0, 90.0, 96.0, 92.0, 106.0, 106.0, 113.0, 127.0, 112.0, 118.0, 96.0, 106.0, 114.0, 93.0, 85.0, 74.0, 105.0, 113.0, 97.0, 113.0, 107.0, 97.0, 109.0, 87.0, 89.0, 108.0, 106.0, 87.0, 120.0, 115.0, 109.0, 111.0, 100.0, 114.0, 102.0, 106.0, 94.0, 106.0, 77.0, 124.0, 112.0, 102.0, 104.0, 111.0, 109.0, 125.0, 114.0, 109.0, 120.0, 120.0, 103.0, 107.0, 86.0, 111.0, 95.0, 102.0, 108.0, 78.0, 100.0, 90.0, 107.0, 101.0, 104.0, 119.0, 100.0, 113.0, 110.0, 113.0, 90.0, 101.0, 107.0, 106.0, 111.0, 88.0, 125.0, 93.0, 106.0, 103.0, 116.0, 127.0, 100.0, 84.0, 102.0, 97.0, 97.0, 94.0, 120.0, 109.0, 110.0, 98.0, 97.0, 113.0, 108.0, 106.0, 143.0, 104.0, 111.0, 106.0, 103.0, 99.0, 110.0, 106.0, 130.0, 121.0, 112.0, 103.0, 101.0, 97.0, 115.0, 127.0, 117.0, 116.0, 109.0, 101.0, 129.0, 101.0, 99.0, 112.0, 91.0, 113.0, 104.0, 122.0, 91.0, 120.0, 124.0, 89.0, 106.0, 106.0, 119.0, 101.0, 98.0, 102.0, 129.0, 107.0, 116.0, 126.0, 127.0, 112.0, 86.0, 106.0, 136.0, 135.0, 107.0, 93.0, 102.0, 118.0, 117.0, 104.0, 123.0, 99.0, 114.0, 92.0, 128.0, 92.0, 107.0, 92.0, 124.0, 106.0, 101.0, 112.0, 106.0, 99.0, 107.0, 110.0, 97.0, 108.0, 117.0, 119.0, 102.0, 116.0, 116.0, 118.0, 108.0, 130.0, 116.0, 118.0, 122.0, 105.0, 104.0, 126.0, 123.0, 118.0, 124.0, 126.0, 97.0, 123.0, 133.0, 101.0, 117.0, 114.0, 120.0, 139.0]}, "params-norm": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.15739, 180.15739, 180.15739, 180.15739, 180.15739, 180.15738, 180.15736, 180.15726, 180.15707, 180.15691, 180.15549, 180.15459, 180.15424, 180.15187, 180.15096, 180.15027, 180.14986, 180.14993, 180.15019, 180.15031, 180.15027, 180.14986, 180.14978, 180.15002, 180.15096, 180.15236, 180.15356, 180.15433, 180.15535, 180.15683, 180.15872, 180.16106, 180.16333, 180.16548, 180.16803, 180.17111, 180.17455, 180.1783, 180.18213, 180.18637, 180.19121, 180.19637, 180.20183, 180.20786, 180.21451, 180.22182, 180.22966, 180.23802, 180.24725, 180.25742, 180.2684, 180.28008, 180.29228, 180.30507, 180.31865, 180.33281, 180.34721, 180.36223, 180.37819, 180.39531, 180.41338, 180.43228, 180.45262, 180.47394, 180.49564, 180.51866, 180.54247, 180.56686, 180.59306, 180.6189, 180.64566, 180.6731, 180.70131, 180.72955, 180.75832, 180.78758, 180.81717, 180.84805, 180.8793, 180.91136, 180.94365, 180.97591, 181.00896, 181.04247, 181.07669, 181.11148, 181.14615, 181.18118, 181.2169, 181.25371, 181.29126, 181.32945, 181.36674, 181.40437, 181.4427, 181.4816, 181.51944, 181.5558, 181.59123, 181.62697, 181.66261, 181.69635, 181.73094, 181.76637, 181.8006, 181.83632, 181.87393, 181.91217, 181.95012, 181.9888, 182.0287, 182.06952, 182.11082, 182.15179, 182.19136, 182.23178, 182.27216, 182.31206, 182.35109, 182.39093, 182.43059, 182.47116, 182.51115, 182.55157, 182.59242, 182.63356, 182.67308, 182.71248, 182.75157, 182.79005, 182.8289, 182.86778, 182.90854, 182.9481, 182.98575, 183.02332, 183.0623, 183.0995, 183.13556, 183.17046, 183.20383, 183.23506, 183.26553, 183.2989, 183.33479, 183.37086, 183.40509, 183.44055, 183.47644, 183.51241, 183.54857, 183.58354, 183.61832, 183.65422, 183.69316, 183.73344, 183.77179, 183.80856, 183.84579, 183.88249, 183.91859, 183.95512, 183.99037, 184.02548, 184.063, 184.10135, 184.13824, 184.17474, 184.21408, 184.25304, 184.29404, 184.33496, 184.37621, 184.41531, 184.4537, 184.4928, 184.53014, 184.56731, 184.60611, 184.64619, 184.68703, 184.72823, 184.77042, 184.81314, 184.85387, 184.89021, 184.92393, 184.95621, 184.99136, 185.02664, 185.06209, 185.10019, 185.14125, 185.18129, 185.22131, 185.26175, 185.30276, 185.34607, 185.38876, 185.43182, 185.47507, 185.51636, 185.55836, 185.60168, 185.64523, 185.68893, 185.73134, 185.77113, 185.80952, 185.84686, 185.88496, 185.92491, 185.96541, 186.00458, 186.04584, 186.08769, 186.13078, 186.17444, 186.2169, 186.25897, 186.30052, 186.34146, 186.38252, 186.42355, 186.46315, 186.50108, 186.53908, 186.57777, 186.61641, 186.65698, 186.69749, 186.73779, 186.776, 186.81406, 186.85432, 186.89455, 186.93593, 186.97723, 187.02032, 187.06329, 187.10561, 187.14796, 187.19154, 187.23483, 187.27914, 187.32254, 187.36426, 187.40421, 187.44449, 187.48557, 187.52713, 187.5705, 187.61469, 187.65993, 187.70628, 187.75299, 187.79915, 187.84256, 187.8851, 187.92828, 187.97391, 188.02026, 188.06656, 188.11136, 188.15483, 188.19771, 188.23875, 188.28041, 188.32339, 188.36717, 188.41173, 188.4559, 188.49995, 188.54559, 188.59273, 188.64139, 188.68826, 188.73679, 188.7838, 188.82909, 188.87553, 188.92162, 188.96811, 189.01474, 189.06255, 189.10872, 189.15393, 189.19994, 189.24557, 189.29164, 189.3381, 189.38397, 189.42863, 189.47279, 189.51843, 189.5647, 189.61183, 189.66019, 189.7094, 189.7603, 189.81245, 189.86432, 189.91537, 189.96579, 190.01378, 190.06058, 190.10844, 190.15665, 190.20692, 190.2585, 190.31071, 190.36349, 190.41649, 190.46754, 190.51726, 190.56802, 190.62105, 190.67397, 190.72807, 190.78218, 190.8349, 190.88562, 190.93848, 190.99274, 191.04617, 191.0997, 191.15161, 191.20273, 191.25496, 191.30672, 191.35922, 191.41141, 191.46227, 191.51437, 191.56682, 191.6205, 191.67529, 191.73068, 191.78505, 191.8385, 191.89308, 191.94789, 192.0024, 192.05864, 192.11432, 192.1684, 192.22186, 192.27574, 192.33052, 192.38582, 192.44121, 192.49785, 192.55418, 192.60825, 192.66292, 192.71729, 192.77345, 192.82953, 192.88582, 192.94179, 192.99664, 193.05156, 193.1075, 193.16364, 193.22198, 193.27934, 193.33693, 193.3927, 193.44841, 193.50385, 193.55917, 193.61432, 193.67184, 193.72919, 193.78648, 193.8439, 193.90105, 193.95886, 194.0177, 194.07675, 194.13638, 194.19586, 194.25424, 194.31471, 194.37587, 194.43796, 194.50008, 194.56322, 194.62543, 194.68716, 194.74808, 194.80829, 194.8662, 194.92447, 194.9838, 195.04256, 195.10059, 195.16046, 195.22166, 195.2832]}, "params-norm vs samples": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [180.15739, 180.15739, 180.15739, 180.15739, 180.15739, 180.15738, 180.15736, 180.15726, 180.15707, 180.15691, 180.15549, 180.15459, 180.15424, 180.15187, 180.15096, 180.15027, 180.14986, 180.14993, 180.15019, 180.15031, 180.15027, 180.14986, 180.14978, 180.15002, 180.15096, 180.15236, 180.15356, 180.15433, 180.15535, 180.15683, 180.15872, 180.16106, 180.16333, 180.16548, 180.16803, 180.17111, 180.17455, 180.1783, 180.18213, 180.18637, 180.19121, 180.19637, 180.20183, 180.20786, 180.21451, 180.22182, 180.22966, 180.23802, 180.24725, 180.25742, 180.2684, 180.28008, 180.29228, 180.30507, 180.31865, 180.33281, 180.34721, 180.36223, 180.37819, 180.39531, 180.41338, 180.43228, 180.45262, 180.47394, 180.49564, 180.51866, 180.54247, 180.56686, 180.59306, 180.6189, 180.64566, 180.6731, 180.70131, 180.72955, 180.75832, 180.78758, 180.81717, 180.84805, 180.8793, 180.91136, 180.94365, 180.97591, 181.00896, 181.04247, 181.07669, 181.11148, 181.14615, 181.18118, 181.2169, 181.25371, 181.29126, 181.32945, 181.36674, 181.40437, 181.4427, 181.4816, 181.51944, 181.5558, 181.59123, 181.62697, 181.66261, 181.69635, 181.73094, 181.76637, 181.8006, 181.83632, 181.87393, 181.91217, 181.95012, 181.9888, 182.0287, 182.06952, 182.11082, 182.15179, 182.19136, 182.23178, 182.27216, 182.31206, 182.35109, 182.39093, 182.43059, 182.47116, 182.51115, 182.55157, 182.59242, 182.63356, 182.67308, 182.71248, 182.75157, 182.79005, 182.8289, 182.86778, 182.90854, 182.9481, 182.98575, 183.02332, 183.0623, 183.0995, 183.13556, 183.17046, 183.20383, 183.23506, 183.26553, 183.2989, 183.33479, 183.37086, 183.40509, 183.44055, 183.47644, 183.51241, 183.54857, 183.58354, 183.61832, 183.65422, 183.69316, 183.73344, 183.77179, 183.80856, 183.84579, 183.88249, 183.91859, 183.95512, 183.99037, 184.02548, 184.063, 184.10135, 184.13824, 184.17474, 184.21408, 184.25304, 184.29404, 184.33496, 184.37621, 184.41531, 184.4537, 184.4928, 184.53014, 184.56731, 184.60611, 184.64619, 184.68703, 184.72823, 184.77042, 184.81314, 184.85387, 184.89021, 184.92393, 184.95621, 184.99136, 185.02664, 185.06209, 185.10019, 185.14125, 185.18129, 185.22131, 185.26175, 185.30276, 185.34607, 185.38876, 185.43182, 185.47507, 185.51636, 185.55836, 185.60168, 185.64523, 185.68893, 185.73134, 185.77113, 185.80952, 185.84686, 185.88496, 185.92491, 185.96541, 186.00458, 186.04584, 186.08769, 186.13078, 186.17444, 186.2169, 186.25897, 186.30052, 186.34146, 186.38252, 186.42355, 186.46315, 186.50108, 186.53908, 186.57777, 186.61641, 186.65698, 186.69749, 186.73779, 186.776, 186.81406, 186.85432, 186.89455, 186.93593, 186.97723, 187.02032, 187.06329, 187.10561, 187.14796, 187.19154, 187.23483, 187.27914, 187.32254, 187.36426, 187.40421, 187.44449, 187.48557, 187.52713, 187.5705, 187.61469, 187.65993, 187.70628, 187.75299, 187.79915, 187.84256, 187.8851, 187.92828, 187.97391, 188.02026, 188.06656, 188.11136, 188.15483, 188.19771, 188.23875, 188.28041, 188.32339, 188.36717, 188.41173, 188.4559, 188.49995, 188.54559, 188.59273, 188.64139, 188.68826, 188.73679, 188.7838, 188.82909, 188.87553, 188.92162, 188.96811, 189.01474, 189.06255, 189.10872, 189.15393, 189.19994, 189.24557, 189.29164, 189.3381, 189.38397, 189.42863, 189.47279, 189.51843, 189.5647, 189.61183, 189.66019, 189.7094, 189.7603, 189.81245, 189.86432, 189.91537, 189.96579, 190.01378, 190.06058, 190.10844, 190.15665, 190.20692, 190.2585, 190.31071, 190.36349, 190.41649, 190.46754, 190.51726, 190.56802, 190.62105, 190.67397, 190.72807, 190.78218, 190.8349, 190.88562, 190.93848, 190.99274, 191.04617, 191.0997, 191.15161, 191.20273, 191.25496, 191.30672, 191.35922, 191.41141, 191.46227, 191.51437, 191.56682, 191.6205, 191.67529, 191.73068, 191.78505, 191.8385, 191.89308, 191.94789, 192.0024, 192.05864, 192.11432, 192.1684, 192.22186, 192.27574, 192.33052, 192.38582, 192.44121, 192.49785, 192.55418, 192.60825, 192.66292, 192.71729, 192.77345, 192.82953, 192.88582, 192.94179, 192.99664, 193.05156, 193.1075, 193.16364, 193.22198, 193.27934, 193.33693, 193.3927, 193.44841, 193.50385, 193.55917, 193.61432, 193.67184, 193.72919, 193.78648, 193.8439, 193.90105, 193.95886, 194.0177, 194.07675, 194.13638, 194.19586, 194.25424, 194.31471, 194.37587, 194.43796, 194.50008, 194.56322, 194.62543, 194.68716, 194.74808, 194.80829, 194.8662, 194.92447, 194.9838, 195.04256, 195.10059, 195.16046, 195.22166, 195.2832]}, "iteration-time": {"start_step": 0, "end_step": 2000, "step_interval": 5, "values": [30.41341, 2.8046, 2.79928, 2.80445, 2.79909, 2.80635, 2.79849, 2.79809, 2.80876, 2.80642, 2.79859, 2.80408, 2.80282, 2.80528, 2.80514, 2.80807, 2.80806, 2.80751, 2.80996, 2.80978, 2.80663, 2.80424, 2.81097, 2.81307, 2.81122, 2.80264, 2.80542, 2.80789, 2.81202, 2.80175, 2.80699, 2.81063, 2.81844, 2.82302, 2.81854, 2.8107, 2.81902, 2.8157, 2.82159, 2.81915, 2.81816, 2.82321, 2.81751, 2.82121, 2.82517, 2.83278, 2.81862, 2.81687, 2.82205, 2.8171, 2.81951, 2.81838, 2.81328, 2.82805, 2.91883, 2.83795, 2.82853, 2.82715, 2.82978, 2.83004, 2.83565, 2.83193, 2.83679, 2.83184, 2.83322, 2.83292, 2.82436, 2.82807, 2.82713, 2.82297, 2.82207, 2.81925, 2.82219, 2.82388, 2.82547, 2.82046, 2.82554, 2.82609, 2.81973, 2.81555, 2.80902, 2.81328, 2.81723, 2.81808, 2.8209, 2.81658, 2.82868, 2.82046, 2.82766, 2.82547, 2.82306, 2.82434, 2.82165, 2.82182, 2.82079, 2.8171, 2.82456, 2.81695, 2.81958, 2.81888, 2.82274, 2.82232, 2.82111, 2.81589, 2.81554, 2.82411, 2.82116, 2.81529, 2.82499, 2.81696, 2.81507, 2.81149, 2.81848, 2.81732, 2.81615, 2.81512, 2.81829, 2.8116, 2.80978, 2.81506, 2.81764, 2.8198, 2.81632, 2.81606, 2.80897, 2.81568, 2.82245, 2.81885, 2.82606, 2.81987, 2.8158, 2.82143, 2.8193, 2.82472, 2.81111, 2.81631, 2.83592, 2.81315, 2.82779, 2.82235, 2.83714, 2.8297, 2.837, 2.83586, 2.83284, 2.83636, 2.83258, 2.83915, 2.83419, 2.83824, 2.84049, 2.84197, 2.84072, 2.83281, 2.82944, 2.8375, 2.81702, 2.84669, 2.82923, 2.81781, 2.82019, 2.82199, 2.81611, 2.82377, 2.82298, 2.82195, 2.81502, 2.81982, 2.8244, 2.83221, 2.82765, 2.81874, 2.82405, 2.81662, 2.82101, 2.8221, 2.81703, 2.81771, 2.81876, 2.81927, 2.8219, 2.81857, 2.82075, 2.8191, 2.82229, 2.82063, 2.82301, 2.82242, 2.82223, 2.81908, 2.82481, 2.82407, 2.82328, 2.82304, 2.8156, 2.8223, 2.8283, 2.82746, 2.83015, 2.82908, 2.79797, 2.79998, 2.78923, 2.79503, 2.80833, 2.79099, 2.78989, 2.78911, 2.78508, 2.78213, 2.78209, 2.79677, 2.78643, 2.78646, 2.78817, 2.77762, 2.78837, 2.78968, 2.78321, 2.78471, 2.78732, 2.79108, 2.78484, 2.79823, 2.78713, 2.78768, 2.78784, 2.78488, 2.7883, 2.78899, 2.79726, 2.78764, 2.79575, 2.7903, 2.7943, 2.78923, 2.79105, 2.78913, 2.78266, 2.78538, 2.78833, 2.79805, 2.78908, 2.79905, 2.79128, 2.79609, 2.79756, 2.78663, 2.79377, 2.83553, 2.82821, 2.82975, 2.82985, 2.8276, 2.83102, 2.82461, 2.83883, 2.82299, 2.82069, 2.82305, 2.81459, 2.82648, 2.82175, 2.82728, 2.82733, 2.82099, 2.83858, 2.83126, 2.83115, 2.82847, 2.83258, 2.83579, 2.83969, 2.83857, 2.86059, 2.84207, 2.84007, 2.84684, 2.84306, 2.84137, 2.84087, 2.79807, 2.79644, 2.79588, 2.79211, 2.79479, 2.80066, 2.79173, 2.79944, 2.79749, 2.80704, 2.79981, 2.79552, 2.79711, 2.7928, 2.79311, 2.78965, 2.78698, 2.78443, 2.78879, 2.79821, 2.79383, 2.79253, 2.79447, 2.78491, 2.77925, 2.78353, 2.78445, 2.79082, 2.79857, 2.80414, 2.80257, 2.78642, 2.78648, 2.78739, 2.78471, 2.78001, 2.78196, 2.78327, 2.78431, 2.791, 2.78454, 2.78713, 2.78803, 2.78024, 2.776, 2.77716, 2.78213, 2.78774, 2.78732, 2.78532, 2.78606, 2.78414, 2.77758, 2.78443, 2.77071, 2.77741, 2.78603, 2.78774, 2.78521, 2.78444, 2.78878, 2.774, 2.78293, 2.78129, 2.78025, 2.78828, 2.78815, 2.78075, 2.78504, 2.77911, 2.77515, 2.77671, 2.77649, 2.88175, 2.77346, 2.78223, 2.78354, 2.77649, 2.78232, 2.77496, 2.78767, 2.7835, 2.77767, 2.7876, 2.78256, 2.77263, 2.77761, 2.77618, 2.782, 2.78046, 2.7906, 2.78832, 2.78117, 2.77888, 2.79122, 2.79084, 2.78287, 2.77695, 2.77599, 2.78415, 2.77982, 2.77929, 2.77879, 2.77575, 2.77152, 2.77167, 2.78528, 2.77604, 2.785, 2.78948, 2.7772, 2.78592, 2.77735, 2.77812, 2.80061, 2.78402, 2.79223, 2.78189, 2.78928]}, "lm loss validation": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60622]}, "lm loss validation vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [5.60622]}, "lm loss validation ppl": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [272.11401]}, "lm loss validation ppl vs samples": {"start_step": 0, "end_step": 4, "step_interval": 5, "values": [272.11401]}}
\ No newline at end of file
diff --git a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/model_config.yaml
index 0d282c7ec9..aa529c3316 100644
--- a/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/model_config.yaml
+++ b/tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp4_pp2_fp8_tp_pp/model_config.yaml
@@ -1,6 +1,7 @@
 ENV_VARS:
   CUDA_DEVICE_MAX_CONNECTIONS: 1
   NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NVTE_FUSED_ATTN: 0
   NCCL_ALGO: Tree
   CUBLAS_WORKSPACE_CONFIG: :4096:8
 MODEL_ARGS:
@@ -45,7 +46,7 @@ MODEL_ARGS:
   --fp8-amax-history-len: 1024
   --fp8-amax-compute-algo: max
   --attention-softmax-in-fp32: true
-  --ckpt-format: true
+  --ckpt-format: torch_dist
   --data-cache-path: ${DATA_CACHE_PATH}
   --bf16: true
-TEST_TYPE: regular
\ No newline at end of file
+TEST_TYPE: regular
diff --git a/tests/unit_tests/dist_checkpointing/utils.py b/tests/unit_tests/dist_checkpointing/utils.py
index e4a007aa75..33220d2801 100644
--- a/tests/unit_tests/dist_checkpointing/utils.py
+++ b/tests/unit_tests/dist_checkpointing/utils.py
@@ -55,6 +55,7 @@ def init_basic_mock_args(args, tp, pp, bf16=True):
     args.accumulate_allreduce_grads_in_fp32 = False
     args.overlap_grad_reduce = False
     args.overlap_param_gather_with_optimizer_step = False
+    args.fp8_param_gather = False
     args.use_distributed_optimizer = True
     args.ddp_bucket_size = None
     args.check_for_nan_in_loss_and_grad = False
diff --git a/tests/unit_tests/distributed/test_param_and_grad_buffer.py b/tests/unit_tests/distributed/test_param_and_grad_buffer.py
index a1a821621f..b2a12aff11 100644
--- a/tests/unit_tests/distributed/test_param_and_grad_buffer.py
+++ b/tests/unit_tests/distributed/test_param_and_grad_buffer.py
@@ -6,7 +6,11 @@
 import torch
 
 from megatron.core import parallel_state
-from megatron.core.distributed import DistributedDataParallelConfig, ParamAndGradBuffer
+from megatron.core.distributed import (
+    DistributedDataParallelConfig,
+    ParamAndGradBuffer,
+    partition_buckets,
+)
 from tests.unit_tests.test_utilities import TestModel, Utils
 
 
@@ -36,6 +40,7 @@ def get_model_and_buffers(
     param_to_name = {}
     for name, param in model.named_parameters():
         param_to_name[param] = name
+    param_indices = list(range(len(params)))
 
     param_and_grad_buffer = ParamAndGradBuffer(
         ddp_config,
@@ -46,6 +51,7 @@ def get_model_and_buffers(
         bucket_size=bucket_size,
         param_to_name=param_to_name,
         gradient_scaling_factor=1.0,
+        param_indices=param_indices,
     )
 
     return model, param_and_grad_buffer
@@ -175,6 +181,12 @@ def test_grad_sync(use_distributed_optimizer: bool, overlap_grad_reduce: bool):
         use_distributed_optimizer=use_distributed_optimizer,
         overlap_grad_reduce=overlap_grad_reduce,
     )
+    bucket_groups = partition_buckets([param_and_grad_buffer])
+    param_to_bucket_group = {}
+    for bucket_group in bucket_groups:
+        for param in bucket_group.params:
+            assert param not in param_to_bucket_group
+            param_to_bucket_group[param] = bucket_group
 
     param_and_grad_buffer.grad_data.data.fill_(1.0)
     expected_grad_data_value_after_collective = 1
@@ -183,6 +195,8 @@ def test_grad_sync(use_distributed_optimizer: bool, overlap_grad_reduce: bool):
 
     params = list(model.parameters())
     for i, param in enumerate(params):
+        assert param in param_to_bucket_group
+        bucket_group = param_to_bucket_group[param]
         register_grad_sync_context = (
             contextlib.nullcontext() if overlap_grad_reduce else pytest.raises(AssertionError)
         )
@@ -192,12 +206,12 @@ def test_grad_sync(use_distributed_optimizer: bool, overlap_grad_reduce: bool):
             finish_grad_sync_context = pytest.raises(AssertionError)
 
         with register_grad_sync_context:
-            param_and_grad_buffer.register_grad_ready(param)
+            bucket_group.register_grad_ready(param)
         with finish_grad_sync_context:
             # When overlap_grad_reduce is True, this should throw an assertion error until all
             # params in the model have registered their grad above.
             # When overlap_grad_reduce is False, the collective is forced through.
-            param_and_grad_buffer.finish_grad_sync()
+            bucket_group.finish_grad_sync()
 
         expected_grad_data_value = expected_grad_data_value_after_collective
         if overlap_grad_reduce and i < (len(params) - 1):