ADLR/megatron-lm!3067 - ci: Update golden values after !2668

Author: ko3n1g

model_config.yaml

@@ -0,0 +1,47 @@
+ENV_VARS:
+  CUDA_DEVICE_MAX_CONNECTIONS: 1
+  NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0
+  NCCL_ALGO: Ring
+  CUBLAS_WORKSPACE_CONFIG: :4096:8
+TEST_TYPE: frozen-start
+MODEL_ARGS:
+  --tiktoken-pattern: v2
+  --use-mcore-models: true
+  --tokenizer-type: TikTokenizer
+  --tokenizer-model: ${TOKENIZER_MODEL}
+  --auto-detect-ckpt-format: true
+  --max-tokens-to-oom: 3600000
+  --inference-max-seq-length: 4096
+  --attention-backend: flash
+  --use-checkpoint-args: true
+  --micro-batch-size: 1
+  --no-load-optim: true
+  --no-use-tokenizer-model-from-checkpoint-args: true
+  --timing-log-level: 2
+  --load: ${CHECKPOINT_LOAD_PATH}
+  --distributed-backend: nccl
+  --log-interval: 1
+  --transformer-impl: transformer_engine
+  --tensor-model-parallel-size: 1
+  --pipeline-model-parallel-size: 1
+  --deterministic-mode: true
+  --ckpt-format: torch_dist
+  --bf16: true
+  --log-memory-to-tensorboard: true
+  --log-num-zeros-in-grad: true
+  --log-validation-ppl-to-tensorboard: true
+  --log-timers-to-tensorboard: true
+  --num-layers: 24
+  --hidden-size: 1152
+  --num-attention-heads: 16
+  --max-position-embeddings: 1024
+  --seq-length: 1024
+  --temperature: 1.0
+  --top_k: 1
+  --return-log-probs: true
+  --num-tokens-to-generate: 30
+  --flash-decode: true
+  --dist-ckpt-strictness: log_unexpected
+  --output-path: ${TENSORBOARD_PATH}
+  --exit-interval: 4
+  --prompts: "Time travel to 2008, and go to a bar or a club or one of the myriad disco-basements on the Lower East Side that does not quite know which of those it is. Dance awkwardly in a room full of other glittered-up nerds, and wait for something to happen, buoyed on the feeling that this is the big swollen heart of life, that this is New York like the movies."

tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_dgx_a100_1N8G/golden_values_lts.json

@@ -1 +1 @@
-{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.49574, "5": 10.48398, "10": 10.49943, "15": 10.4663, "20": 10.44775, "25": 10.34954, "30": 10.17283, "35": 10.0427, "40": 9.9076, "45": 9.7577, "50": 9.67688}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2182.0, "5": 2568.0, "10": 2108.0, "15": 2533.0, "20": 2166.0, "25": 2639.0, "30": 2769.0, "35": 3080.0, "40": 2282.0, "45": 3831.0, "50": 3519.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2300849152.0, "5": 3043311616.0, "10": 3043311616.0, "15": 3043311616.0, "20": 3043311616.0, "25": 3043311616.0, "30": 3043311616.0, "35": 3043311616.0, "40": 3043311616.0, "45": 3043311616.0, "50": 3043311616.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 12.84889, "5": 1.14453, "10": 1.15867, "15": 1.14254, "20": 1.15471, "25": 1.13179, "30": 1.33887, "35": 1.14522, "40": 1.16202, "45": 1.13535, "50": 1.13398}}}
+{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.49574, "5": 10.48398, "10": 10.49943, "15": 10.4663, "20": 10.44775, "25": 10.34954, "30": 10.17283, "35": 10.0427, "40": 9.9076, "45": 9.7577, "50": 9.67688}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2182.0, "5": 2568.0, "10": 2108.0, "15": 2533.0, "20": 2166.0, "25": 2639.0, "30": 2769.0, "35": 3080.0, "40": 2282.0, "45": 3831.0, "50": 3519.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2300849152.0, "5": 3043311616.0, "10": 3043311616.0, "15": 3043311616.0, "20": 3043311616.0, "25": 3043311616.0, "30": 3043311616.0, "35": 3043311616.0, "40": 3043311616.0, "45": 3043311616.0, "50": 3043311616.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 12.27375, "5": 1.16776, "10": 1.17722, "15": 1.16736, "20": 1.17298, "25": 1.17438, "30": 1.38407, "35": 1.16408, "40": 1.16845, "45": 1.17212, "50": 1.16537}}}

tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_local_spec_dgx_a100_1N8G/golden_values_lts.json

@@ -1 +1 @@
-{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.49566, "5": 10.48418, "10": 10.49947, "15": 10.46646, "20": 10.44788, "25": 10.34978, "30": 10.17275, "35": 10.04282, "40": 9.90773, "45": 9.75782, "50": 9.67689}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2183.0, "5": 2533.0, "10": 2162.0, "15": 2548.0, "20": 2191.0, "25": 2557.0, "30": 2864.0, "35": 2979.0, "40": 2332.0, "45": 3931.0, "50": 3611.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1767237120.0, "5": 1767237120.0, "10": 1767237120.0, "15": 1767237120.0, "20": 1767237120.0, "25": 1767237120.0, "30": 1767237120.0, "35": 1767237120.0, "40": 1767237120.0, "45": 1767237120.0, "50": 1767237120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2313432064.0, "5": 3055894528.0, "10": 3055894528.0, "15": 3055894528.0, "20": 3055894528.0, "25": 3055894528.0, "30": 3055894528.0, "35": 3055894528.0, "40": 3055894528.0, "45": 3055894528.0, "50": 3055894528.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 18.09734, "5": 1.25804, "10": 1.21272, "15": 1.20685, "20": 1.38962, "25": 1.39883, "30": 1.20559, "35": 1.32546, "40": 1.26983, "45": 1.23104, "50": 1.19999}}}
+{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.49566, "5": 10.48418, "10": 10.49947, "15": 10.46646, "20": 10.44788, "25": 10.34978, "30": 10.17275, "35": 10.04282, "40": 9.90773, "45": 9.75782, "50": 9.67689}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2183.0, "5": 2533.0, "10": 2162.0, "15": 2548.0, "20": 2191.0, "25": 2557.0, "30": 2864.0, "35": 2979.0, "40": 2332.0, "45": 3931.0, "50": 3611.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1767237120.0, "5": 1767237120.0, "10": 1767237120.0, "15": 1767237120.0, "20": 1767237120.0, "25": 1767237120.0, "30": 1767237120.0, "35": 1767237120.0, "40": 1767237120.0, "45": 1767237120.0, "50": 1767237120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 2313432064.0, "5": 3055894528.0, "10": 3055894528.0, "15": 3055894528.0, "20": 3055894528.0, "25": 3055894528.0, "30": 3055894528.0, "35": 3055894528.0, "40": 3055894528.0, "45": 3055894528.0, "50": 3055894528.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 12.05431, "5": 1.18966, "10": 1.17532, "15": 1.17174, "20": 1.17938, "25": 1.35744, "30": 1.17707, "35": 1.17213, "40": 1.18767, "45": 1.1769, "50": 1.18145}}}

tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_dgx_a100_1N8G/golden_values_lts.json

@@ -1 +1 @@
-{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49574, "5": 10.48398, "10": 10.49943, "15": 10.4663, "20": 10.44775, "25": 10.34954, "30": 10.17283, "35": 10.0427, "40": 9.9076, "45": 9.7577, "50": 9.67688, "55": 9.55375, "60": 9.4546, "65": 9.42141, "70": 9.30109, "75": 9.32202, "80": 9.26199, "85": 9.29667, "90": 9.2334, "95": 9.23801, "100": 9.10602}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2182.0, "5": 2568.0, "10": 2108.0, "15": 2533.0, "20": 2166.0, "25": 2639.0, "30": 2769.0, "35": 3080.0, "40": 2282.0, "45": 3831.0, "50": 3519.0, "55": 3692.0, "60": 2614.0, "65": 3344.0, "70": 4018.0, "75": 4983.0, "80": 3679.0, "85": 4082.0, "90": 4634.0, "95": 4487.0, "100": 3079.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0, "55": 1754654208.0, "60": 1754654208.0, "65": 1754654208.0, "70": 1754654208.0, "75": 1754654208.0, "80": 1754654208.0, "85": 1754654208.0, "90": 1754654208.0, "95": 1754654208.0, "100": 1754654208.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2300849152.0, "5": 3043311616.0, "10": 3043311616.0, "15": 3043311616.0, "20": 3043311616.0, "25": 3043311616.0, "30": 3043311616.0, "35": 3043311616.0, "40": 3043311616.0, "45": 3043311616.0, "50": 3043311616.0, "55": 3043311616.0, "60": 3043311616.0, "65": 3043311616.0, "70": 3043311616.0, "75": 3043311616.0, "80": 3043311616.0, "85": 3043311616.0, "90": 3043831296.0, "95": 3043831296.0, "100": 3043831296.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 16.75433, "5": 1.17309, "10": 1.16842, "15": 1.17296, "20": 1.15562, "25": 1.16985, "30": 1.32568, "35": 1.1614, "40": 1.34213, "45": 1.1597, "50": 1.15711, "55": 1.21795, "60": 1.16711, "65": 1.16097, "70": 1.15412, "75": 1.15824, "80": 1.15542, "85": 1.1511, "90": 1.16694, "95": 1.17311, "100": 1.48866}}}
+{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49574, "5": 10.48398, "10": 10.49943, "15": 10.4663, "20": 10.44775, "25": 10.34954, "30": 10.17283, "35": 10.0427, "40": 9.9076, "45": 9.7577, "50": 9.67688, "55": 9.55375, "60": 9.4546, "65": 9.42141, "70": 9.30109, "75": 9.32202, "80": 9.26199, "85": 9.29667, "90": 9.2334, "95": 9.23801, "100": 9.10602}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2182.0, "5": 2568.0, "10": 2108.0, "15": 2533.0, "20": 2166.0, "25": 2639.0, "30": 2769.0, "35": 3080.0, "40": 2282.0, "45": 3831.0, "50": 3519.0, "55": 3692.0, "60": 2614.0, "65": 3344.0, "70": 4018.0, "75": 4983.0, "80": 3679.0, "85": 4082.0, "90": 4634.0, "95": 4487.0, "100": 3079.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1754654208.0, "5": 1754654208.0, "10": 1754654208.0, "15": 1754654208.0, "20": 1754654208.0, "25": 1754654208.0, "30": 1754654208.0, "35": 1754654208.0, "40": 1754654208.0, "45": 1754654208.0, "50": 1754654208.0, "55": 1754654208.0, "60": 1754654208.0, "65": 1754654208.0, "70": 1754654208.0, "75": 1754654208.0, "80": 1754654208.0, "85": 1754654208.0, "90": 1754654208.0, "95": 1754654208.0, "100": 1754654208.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2300849152.0, "5": 3043311616.0, "10": 3043311616.0, "15": 3043311616.0, "20": 3043311616.0, "25": 3043311616.0, "30": 3043311616.0, "35": 3043311616.0, "40": 3043311616.0, "45": 3043311616.0, "50": 3043311616.0, "55": 3043311616.0, "60": 3043311616.0, "65": 3043311616.0, "70": 3043311616.0, "75": 3043311616.0, "80": 3043311616.0, "85": 3043311616.0, "90": 3043311616.0, "95": 3043311616.0, "100": 3043311616.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 12.04821, "5": 1.17042, "10": 1.16137, "15": 1.14597, "20": 1.13585, "25": 1.13875, "30": 1.30978, "35": 1.15335, "40": 1.20199, "45": 1.14529, "50": 1.14959, "55": 1.18763, "60": 1.14714, "65": 1.15404, "70": 1.16403, "75": 1.14577, "80": 1.14934, "85": 1.15529, "90": 1.14761, "95": 1.14382, "100": 1.3655}}}

tests/functional_tests/test_cases/bert/bert_mr_mcore_tp2_pp2_resume_torch_dist_local_spec_dgx_a100_1N8G/golden_values_lts.json

@@ -1 +1 @@
-{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49566, "5": 10.48418, "10": 10.49947, "15": 10.46646, "20": 10.44788, "25": 10.34978, "30": 10.17275, "35": 10.04282, "40": 9.90773, "45": 9.75782, "50": 9.67689, "55": 9.55382, "60": 9.45468, "65": 9.42164, "70": 9.30116, "75": 9.32219, "80": 9.26192, "85": 9.29673, "90": 9.23346, "95": 9.238, "100": 9.1061}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2183.0, "5": 2533.0, "10": 2162.0, "15": 2548.0, "20": 2191.0, "25": 2557.0, "30": 2864.0, "35": 2979.0, "40": 2332.0, "45": 3931.0, "50": 3611.0, "55": 3750.0, "60": 2647.0, "65": 3396.0, "70": 3869.0, "75": 4912.0, "80": 3687.0, "85": 4182.0, "90": 4677.0, "95": 4397.0, "100": 3212.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1767237120.0, "5": 1767237120.0, "10": 1767237120.0, "15": 1767237120.0, "20": 1768285696.0, "25": 1767237120.0, "30": 1767237120.0, "35": 1767237120.0, "40": 1767237120.0, "45": 1767237120.0, "50": 1767237120.0, "55": 1767237120.0, "60": 1767237120.0, "65": 1767237120.0, "70": 1767237120.0, "75": 1767237120.0, "80": 1767237120.0, "85": 1767237120.0, "90": 1767237120.0, "95": 1767237120.0, "100": 1767237120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2313432064.0, "5": 3055894528.0, "10": 3055894528.0, "15": 3055894528.0, "20": 3055894528.0, "25": 3055894528.0, "30": 3055894528.0, "35": 3055894528.0, "40": 3055894528.0, "45": 3055894528.0, "50": 3055894528.0, "55": 3055894528.0, "60": 3055894528.0, "65": 3055894528.0, "70": 3055894528.0, "75": 3055894528.0, "80": 3055894528.0, "85": 3055894528.0, "90": 3055894528.0, "95": 3055894528.0, "100": 3055894528.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 13.385, "5": 1.19908, "10": 1.27537, "15": 1.21082, "20": 1.22933, "25": 1.38868, "30": 1.20436, "35": 1.20556, "40": 1.2017, "45": 1.20062, "50": 1.16251, "55": 1.16283, "60": 1.15428, "65": 1.1485, "70": 1.15494, "75": 1.14907, "80": 1.15664, "85": 1.15071, "90": 1.15224, "95": 1.15915, "100": 1.15944}}}
+{"lm loss": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 10.49566, "5": 10.48418, "10": 10.49947, "15": 10.46646, "20": 10.44788, "25": 10.34978, "30": 10.17275, "35": 10.04282, "40": 9.90773, "45": 9.75782, "50": 9.67689, "55": 9.55382, "60": 9.45468, "65": 9.42164, "70": 9.30116, "75": 9.32219, "80": 9.26192, "85": 9.29673, "90": 9.23346, "95": 9.238, "100": 9.1061}}, "num-zeros": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2183.0, "5": 2533.0, "10": 2162.0, "15": 2548.0, "20": 2191.0, "25": 2557.0, "30": 2864.0, "35": 2979.0, "40": 2332.0, "45": 3931.0, "50": 3611.0, "55": 3750.0, "60": 2647.0, "65": 3396.0, "70": 3869.0, "75": 4912.0, "80": 3687.0, "85": 4182.0, "90": 4677.0, "95": 4397.0, "100": 3212.0}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 1767237120.0, "5": 1768285696.0, "10": 1767237120.0, "15": 1767237120.0, "20": 1767237120.0, "25": 1768285696.0, "30": 1767237120.0, "35": 1767237120.0, "40": 1767237120.0, "45": 1767237120.0, "50": 1767237120.0, "55": 1767237120.0, "60": 1767237120.0, "65": 1767237120.0, "70": 1767237120.0, "75": 1767237120.0, "80": 1767237120.0, "85": 1767237120.0, "90": 1767237120.0, "95": 1767237120.0, "100": 1767237120.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 2313432064.0, "5": 3055894528.0, "10": 3055894528.0, "15": 3055894528.0, "20": 3055894528.0, "25": 3055894528.0, "30": 3055894528.0, "35": 3055894528.0, "40": 3055894528.0, "45": 3055894528.0, "50": 3055894528.0, "55": 3055894528.0, "60": 3055894528.0, "65": 3055894528.0, "70": 3055894528.0, "75": 3055894528.0, "80": 3055894528.0, "85": 3055894528.0, "90": 3055894528.0, "95": 3055894528.0, "100": 3055894528.0}}, "iteration-time": {"start_step": 1, "end_step": 100, "step_interval": 5, "values": {"1": 14.84481, "5": 1.27307, "10": 1.23221, "15": 1.20069, "20": 1.18788, "25": 1.19693, "30": 1.18416, "35": 1.1848, "40": 1.19561, "45": 1.20472, "50": 1.19232, "55": 1.1978, "60": 1.19225, "65": 1.26274, "70": 1.20824, "75": 1.20479, "80": 1.19807, "85": 1.27802, "90": 1.20241, "95": 1.20248, "100": 1.29984}}}

tests/functional_tests/test_cases/bert/bert_mr_tp1_pp4_vp2_dgx_a100_1N8G/golden_values_lts.json

@@ -1 +1 @@
-{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.54308, "5": 10.54015, "10": 10.54067, "15": 10.56164, "20": 10.54299, "25": 10.53253, "30": 10.45968, "35": 10.31937, "40": 10.18145, "45": 10.03915, "50": 9.91418}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1416466432.0, "5": 1416466432.0, "10": 1416466432.0, "15": 1416466432.0, "20": 2277237760.0, "25": 2276713472.0, "30": 2277237760.0, "35": 2277237760.0, "40": 2277237760.0, "45": 2277237760.0, "50": 2277237760.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 4155522048.0, "5": 4155523072.0, "10": 4155523072.0, "15": 4155523072.0, "20": 5016294400.0, "25": 5016294400.0, "30": 5016294400.0, "35": 5016294400.0, "40": 5016294400.0, "45": 5016294400.0, "50": 5016294400.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 20.47177, "5": 0.8176, "10": 1.10375, "15": 0.8118, "20": 0.82418, "25": 1.02333, "30": 0.79168, "35": 0.81267, "40": 0.94685, "45": 0.79072, "50": 0.76288}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 20512.0, "25": 29663.0, "30": 21646.0, "35": 24318.0, "40": 23486.0, "45": 32603.0, "50": 31901.0}}}
+{"lm loss": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 10.54308, "5": 10.54015, "10": 10.54067, "15": 10.56164, "20": 10.54299, "25": 10.53253, "30": 10.45968, "35": 10.31937, "40": 10.18145, "45": 10.03915, "50": 9.91418}}, "mem-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 1416466432.0, "5": 1416466432.0, "10": 1416466432.0, "15": 1416466432.0, "20": 2277237760.0, "25": 2276713472.0, "30": 2277237760.0, "35": 2277237760.0, "40": 2277237760.0, "45": 2277237760.0, "50": 2277237760.0}}, "mem-max-allocated-bytes": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 4155522048.0, "5": 4155523072.0, "10": 4155523072.0, "15": 4155523072.0, "20": 5016294400.0, "25": 5016294400.0, "30": 5016294400.0, "35": 5016294400.0, "40": 5016294400.0, "45": 5016294400.0, "50": 5016294400.0}}, "iteration-time": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": 15.48584, "5": 0.88474, "10": 1.15052, "15": 0.83058, "20": 0.89747, "25": 1.07634, "30": 0.84226, "35": 0.82046, "40": 1.0039, "45": 0.93385, "50": 0.793}}, "num-zeros": {"start_step": 1, "end_step": 50, "step_interval": 5, "values": {"1": "nan", "5": "nan", "10": "nan", "15": "nan", "20": 20512.0, "25": 29663.0, "30": 21646.0, "35": 24318.0, "40": 23486.0, "45": 32603.0, "50": 31901.0}}}