

"zero_optimization": {
    "stage": 3,
    "offload_optimizer": {
        "device": "nvme",
        "nvme_path": "/local_nvme",
        "pin_memory": true,
        "buffer_count": 4,
        "fast_init": false
    },
    "offload_param": {
        "device": "nvme",
        "nvme_path": "/local_nvme",
        "pin_memory": true,
        "buffer_count": 5,
        "buffer_size": 1e8,
        "max_in_cpu": 1e9
    },
    "aio": {
        "block_size": 262144,
        "queue_depth": 32,
        "thread_count": 1,
        "single_submit": false,
        "overlap_events": true
    },
    "overlap_comm": true,
    "contiguous_gradients": true,
    "sub_group_size": 1e9,
    "reduce_bucket_size": "auto",
    "stage3_prefetch_bucket_size": "auto",
    "stage3_param_persistence_threshold": "auto",
    "stage3_max_live_parameters": 1e9,
    "stage3_max_reuse_distance": 1e9,
    "stage3_gather_16bit_weights_on_model_save": true
},