DPA3 finetune Error #5169

Zch102xjtumse · 2026-01-21T11:21:20Z

Zch102xjtumse
Jan 21, 2026

基于Alloy_tongqi分支进行微调时遇到报错：
Traceback (most recent call last):
File "/opt/deepmd-kit-3.1.0/bin/dp", line 10, in
sys.exit(main())
^^^^^^
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/deepmd/main.py", line 930, in main
deepmd_main(args)
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/torch/distributed/elastic/multiprocessing/errors/init.py", line 355, in wrapper
return f(*args, **kwargs)
^^^^^^^^^^^^^^^^^^
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/deepmd/pt/entrypoints/main.py", line 532, in main
train(
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/deepmd/pt/entrypoints/main.py", line 363, in train
trainer.run()
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/deepmd/pt/train/training.py", line 986, in run
step(step_id)
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/deepmd/pt/train/training.py", line 733, in step
torch.nn.utils.clip_grad_norm_(
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/torch/nn/utils/clip_grad.py", line 34, in no_grad_wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/torch/nn/utils/clip_grad.py", line 215, in clip_grad_norm
total_norm = _get_total_norm(grads, norm_type, error_if_nonfinite, foreach)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/torch/nn/utils/clip_grad.py", line 34, in _no_grad_wrapper
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/opt/deepmd-kit-3.1.0/lib/python3.12/site-packages/torch/nn/utils/clip_grad.py", line 102, in _get_total_norm
raise RuntimeError(
RuntimeError: The total norm of order 2.0 for gradients from parameters is non-finite, so it cannot be clipped. To disable this error and scale the gradients by the non-finite norm anyway, set error_if_nonfinite=False
以下是我的input文件：
{
"_comment": "that's all",
"model": {
"type_map": [
"Cr",
"Co",
"Ni",
"H"
],
"descriptor": {
"type": "dpa3",
"repflow": {
"n_dim": 128,
"e_dim": 64,
"a_dim": 32,
"nlayers": 16,
"e_rcut": 6.0,
"e_rcut_smth": 5.3,
"e_sel": 1200,
"a_rcut": 4.0,
"a_rcut_smth": 3.5,
"a_sel": 300,
"axis_neuron": 4,
"fix_stat_std": 0.3,
"a_compress_rate": 1,
"a_compress_e_rate": 2,
"a_compress_use_split": true,
"update_angle": true,
"smooth_edge_update": true,
"use_dynamic_sel": true,
"sel_reduce_factor": 10.0,
"use_exp_switch": true,
"update_style": "res_residual",
"update_residual": 0.1,
"update_residual_init": "const"
},
"activation_function": "silut:3.0",
"use_tebd_bias": false,
"precision": "float32",
"concat_output_tebd": false
},
"fitting_net": {
"neuron": [
240,
240,
240
],
"dim_case_embd": 31,
"resnet_dt": true,
"precision": "float32",
"activation_function": "silut:3.0",
"seed": 1,
"_comment": " that's all"
},
"_comment": " that's all"
},
"learning_rate": {
"type": "exp",
"decay_steps": 5000,
"start_lr": 0.0002,
"stop_lr": 1e-05,
"_comment": "that's all"
},
"loss": {
"type": "ener",
"start_pref_e": 0.2,
"limit_pref_e": 20,
"start_pref_f": 200,
"limit_pref_f": 60,
"start_pref_v": 0.02,
"limit_pref_v": 1,
"_comment": " that's all"
},
"training": {
"stat_file": "./dpa3_stat.hdf5",
"training_data": {
"systems": [
"./data/trainingsets/AIMD-Hmelt",
"./data/trainingsets/AIMDmelt",
"./data/trainingsets/int-relax/oct-data",
"./data/trainingsets/int-relax/tet-data",
"./data/trainingsets/relax-vac/relax-vac-ni/coco0.3",
"./data/trainingsets/relax-vac/relax-vac-ni/cocon0.3",
"./data/trainingsets/relax-vac/relax-vac-co/coco0.3",
"./data/trainingsets/relax-vac/relax-vac-co/cocon0.3",
"./data/trainingsets/relax-vac/relax-vac-co/cocr0.3",
"./data/trainingsets/relax-vac/relax-vac-co/cocrn0.3",
"./data/trainingsets/relax-vac/relax-vac-co/crcr0.3",
"./data/trainingsets/relax-vac/relax-vac-co/crcrn0.3",
"./data/trainingsets/relax-vac/relax-vac-cr/cocon0.3",
"./data/trainingsets/relax-vac/relax-vac-cr/cocr0.3",
"./data/trainingsets/relax-vac/relax-vac-cr/cocrn0.3",
"./data/trainingsets/relax-vac/relax-vac-cr/crcr0.3",
"./data/trainingsets/relax-vac/relax-vac-cr/crcrn0.3",
"./data/trainingsets/add-1/test_1-merged_4_high_error",
"./data/trainingsets/add-1/test_INT-merged_1_high_error",
"./data/trainingsets/add-1/test_INT-merged_2_high_error",
"./data/trainingsets/add-1/test_INT-merged_3_high_error",
"./data/trainingsets/add-1/test_INT-merged_4_high_error",
"./data/trainingsets/add-1/test_INT-merged_5_high_error",
"./data/trainingsets/add-1/test_dpmc-merged_1_high_error",
"./data/trainingsets/add-1/test_s-merged_14_high_error",
"./data/trainingsets/add-1/test_s-merged_16_high_error",
"./data/trainingsets/add-1/test_s-merged_18_high_error",
"./data/trainingsets/add-csro/1",
"./data/trainingsets/add-csro/2",
"./data/trainingsets/add-csro/3",
"./data/trainingsets/add-csro/4",
"./data/trainingsets/add-csro/5",
"./data/trainingsets/add-csro/6",
"./data/trainingsets/add-csro/7",
"./data/trainingsets/add-csro2/1",
"./data/trainingsets/add-csro2/2",
"./data/trainingsets/add-csro2/3",
"./data/trainingsets/add-csro2/4",
"./data/trainingsets/add-csro2/5",
"./data/trainingsets/add-csro2/6",
"./data/trainingsets/add-csro2/7",
"./data/trainingsets/dp/gsfe-H-1",
"./data/trainingsets/dp/gsfe-H-2",
"./data/trainingsets/dp/gsfe-H-3",
"./data/trainingsets/dp/lc",
"./data/trainingsets/dp/oct",
"./data/trainingsets/dp/tet",
"./data/trainingsets/vacancy-traverse/1",
"./data/trainingsets/vacancy-traverse/2",
"./data/trainingsets/vacancy-traverse/3",
"./data/trainingsets/vacancy-traverse/4",
"./data/trainingsets/EV",
"./data/trainingsets/perturb",
"./data/trainingsets/vacancy-H/Ni",
"./data/trainingsets/vacancy-H/Co",
"./data/trainingsets/vacancy-H/Cr",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_1",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_10",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_11",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_12",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_13",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_14",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_15",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_16",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_17",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_18",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_2",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_3",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_4",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_5",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_6",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_7",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_8",
"./data/trainingsets/train/1-merged_10_train_4_train/small-train_1-merged_10_train_train_1-merged_9",
"./data/trainingsets/train/INT-merged_10_train_4_train/small-train_INT-merged_10_train_train_INT-merged_1",
"./data/trainingsets/train/INT-merged_10_train_4_train/small-train_INT-merged_10_train_train_INT-merged_2",
"./data/trainingsets/train/INT-merged_10_train_4_train/small-train_INT-merged_10_train_train_INT-merged_3",
"./data/trainingsets/train/INT-merged_10_train_4_train/small-train_INT-merged_10_train_train_INT-merged_4",
"./data/trainingsets/train/INT-merged_10_train_4_train/small-train_INT-merged_10_train_train_INT-merged_5",
"./data/trainingsets/train/add-merged_10_train_4_train/small-train_add-merged_10_train_train_add-merged_4",
"./data/trainingsets/train/add-merged_10_train_4_train/small-train_add-merged_10_train_train_add-merged_5",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_1",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_10",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_2",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_3",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_4",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_5",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_6",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_7",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_8",
"./data/trainingsets/train/dpmc-merged_10_train_4_train/small-train_dpmc-merged_10_train_train_dpmc-merged_9",
"./data/trainingsets/train/randommc-merged_10_train_4_train/small-train_randommc-merged_10_train_train_randommc-merged_1",
"./data/trainingsets/train/randommc-merged_10_train_4_train/small-train_randommc-merged_10_train_train_randommc-merged_2",
"./data/trainingsets/train/randommc-merged_10_train_4_train/small-train_randommc-merged_10_train_train_randommc-merged_3",
"./data/trainingsets/train/randommc-merged_10_train_4_train/small-train_randommc-merged_10_train_train_randommc-merged_4",
"./data/trainingsets/train/randommc-merged_10_train_4_train/small-train_randommc-merged_10_train_train_randommc-merged_5",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_1",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_10",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_11",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_12",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_13",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_14",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_15",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_16",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_17",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_18",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_19",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_20",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_21",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_22",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_3",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_4",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_5",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_6",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_7",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_8",
"./data/trainingsets/train/s-merged_10_train_4_train/small-train_s-merged_10_train_train_s-merged_9",
"./data/trainingsets/train-gsfe-1/1",
"./data/trainingsets/train-gsfe-1/2",
"./data/trainingsets/train-gsfe-1/3",
"./data/trainingsets/train-gsfe-1/4",
"./data/trainingsets/train-gsfe-1/5",
"./data/trainingsets/train-gsfe-1/6",
"./data/trainingsets/train-gsfe-1/7",
"./data/trainingsets/train-gsfe-1/8",
"./data/trainingsets/train-gsfe-2/1",
"./data/trainingsets/train-gsfe-2/2",
"./data/trainingsets/train-gsfe-2/3",
"./data/trainingsets/train-gsfe-2/4",
"./data/trainingsets/train-gsfe-2/5",
"./data/trainingsets/train-gsfe-2/6",
"./data/trainingsets/train-gsfe-2/7",
"./data/trainingsets/train-gsfe-2/8",
"./data/trainingsets/dp-interstitial/1",
"./data/trainingsets/dp-interstitial/2",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-22V-1NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-22V-2NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-22V-4NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-22V-5NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-23V-112-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-23V-113-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-23V-226-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-2V-1NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-2V-2NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-2V-4NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-2V-5NN-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-3V-112-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-3V-113-deepmd",
"./data/trainingsets/dp-vacancy-relax/OUTCAR-3V-226-deepmd"
],
"batch_size": 1,
"_comment": "that's all",
"auto_prob_style": "prob_sys_size;0:2:0.125;2:18:0.75;18:149:0.125"
},
"validation_data": {
"systems": [
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_1",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_10",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_11",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_12",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_13",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_14",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_15",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_16",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_17",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_18",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_2",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_3",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_4",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_5",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_6",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_7",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_8",
"./data/validationsets/V_sets/1-merged_10_valid/test_1-merged_9",
"./data/validationsets/V_sets/INT-merged_10_valid/test_INT-merged_1",
"./data/validationsets/V_sets/INT-merged_10_valid/test_INT-merged_2",
"./data/validationsets/V_sets/INT-merged_10_valid/test_INT-merged_3",
"./data/validationsets/V_sets/INT-merged_10_valid/test_INT-merged_4",
"./data/validationsets/V_sets/INT-merged_10_valid/test_INT-merged_5",
"./data/validationsets/V_sets/add-merged_10_valid/test_add-merged_4",
"./data/validationsets/V_sets/add-merged_10_valid/test_add-merged_5",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_1",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_10",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_2",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_3",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_4",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_5",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_6",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_7",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_8",
"./data/validationsets/V_sets/dpmc-merged_10_valid/test_dpmc-merged_9",
"./data/validationsets/V_sets/randommc-merged_10_valid/test_randommc-merged_1",
"./data/validationsets/V_sets/randommc-merged_10_valid/test_randommc-merged_2",
"./data/validationsets/V_sets/randommc-merged_10_valid/test_randommc-merged_3",
"./data/validationsets/V_sets/randommc-merged_10_valid/test_randommc-merged_4",
"./data/validationsets/V_sets/randommc-merged_10_valid/test_randommc-merged_5",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_1",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_10",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_11",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_12",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_13",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_14",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_15",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_16",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_17",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_18",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_19",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_20",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_21",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_22",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_3",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_4",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_5",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_6",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_7",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_8",
"./data/validationsets/V_sets/s-merged_10_valid/test_s-merged_9",
"./data/validationsets/test-gsfe-1/1",
"./data/validationsets/test-gsfe-1/2",
"./data/validationsets/test-gsfe-1/3",
"./data/validationsets/test-gsfe-1/4",
"./data/validationsets/test-gsfe-1/5",
"./data/validationsets/test-gsfe-1/6",
"./data/validationsets/test-gsfe-1/7",
"./data/validationsets/test-gsfe-1/8",
"./data/validationsets/test-gsfe-2/1",
"./data/validationsets/test-gsfe-2/2",
"./data/validationsets/test-gsfe-2/3",
"./data/validationsets/test-gsfe-2/4",
"./data/validationsets/test-gsfe-2/5",
"./data/validationsets/test-gsfe-2/6",
"./data/validationsets/test-gsfe-2/7",
"./data/validationsets/test-gsfe-2/8",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_1",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_10",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_11",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_12",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_13",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_14",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_15",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_16",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_17",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_18",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_2",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_3",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_4",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_5",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_6",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_7",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_8",
"./data/validationsets/valid/1-merged_10_train_4_valid/small-test_1-merged_10_train_train_1-merged_9",
"./data/validationsets/valid/INT-merged_10_train_4_valid/small-test_INT-merged_10_train_train_INT-merged_1",
"./data/validationsets/valid/INT-merged_10_train_4_valid/small-test_INT-merged_10_train_train_INT-merged_2",
"./data/validationsets/valid/INT-merged_10_train_4_valid/small-test_INT-merged_10_train_train_INT-merged_3",
"./data/validationsets/valid/INT-merged_10_train_4_valid/small-test_INT-merged_10_train_train_INT-merged_4",
"./data/validationsets/valid/INT-merged_10_train_4_valid/small-test_INT-merged_10_train_train_INT-merged_5",
"./data/validationsets/valid/add-merged_10_train_4_valid/small-test_add-merged_10_train_train_add-merged_4",
"./data/validationsets/valid/add-merged_10_train_4_valid/small-test_add-merged_10_train_train_add-merged_5",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_1",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_10",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_2",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_3",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_4",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_5",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_6",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_7",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_8",
"./data/validationsets/valid/dpmc-merged_10_train_4_valid/small-test_dpmc-merged_10_train_train_dpmc-merged_9",
"./data/validationsets/valid/randommc-merged_10_train_4_valid/small-test_randommc-merged_10_train_train_randommc-merged_1",
"./data/validationsets/valid/randommc-merged_10_train_4_valid/small-test_randommc-merged_10_train_train_randommc-merged_2",
"./data/validationsets/valid/randommc-merged_10_train_4_valid/small-test_randommc-merged_10_train_train_randommc-merged_3",
"./data/validationsets/valid/randommc-merged_10_train_4_valid/small-test_randommc-merged_10_train_train_randommc-merged_4",
"./data/validationsets/valid/randommc-merged_10_train_4_valid/small-test_randommc-merged_10_train_train_randommc-merged_5",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_1",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_10",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_11",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_12",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_13",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_14",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_15",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_16",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_17",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_18",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_19",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_20",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_21",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_22",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_3",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_4",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_5",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_6",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_7",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_8",
"./data/validationsets/valid/s-merged_10_train_4_valid/small-test_s-merged_10_train_train_s-merged_9"
],
"batch_size": 1,
"_comment": "that's all"
},
"numb_steps": 200000,
"warmup_steps": 0,
"gradient_max_norm": 5.0,
"seed": 10,
"disp_file": "lcurve.out",
"disp_freq": 100,
"save_freq": 2000,
"_comment": "that's all"
}
}
请各位老师帮忙看看原因，设置是否存在问题

@Zch102xjtumse · 2026-01-21T11:28:53Z

dosubot[bot]
bot Jan 21, 2026

Hi @Zch102xjtumse! I'm Dosu and I’m helping the deepmd-kit team.

你遇到的“non-finite gradient norm”报错，通常是因为模型梯度中出现了NaN或Inf，常见原因有：训练数据或stat_file中有异常值（NaN/Inf/极端值）、模型参数设置不合理、或者DPA3实现本身的数值不稳定。

建议排查和调整：

检查数据和stat_file：用numpy/h5py等工具检查所有训练数据（energy.npy、force.npy等）和./dpa3_stat.hdf5中是否有NaN、Inf或极端异常值。例如：
```
import numpy as np
data = np.load('force.npy')
print(np.isnan(data).any(), np.isinf(data).any(), np.max(np.abs(data)))
```
HDF5文件可用h5py检查。
调整loss权重：你的start_pref_f=200高于官方推荐（DPA3建议start_pref_f=100，limit_pref_f=60），过大容易导致训练不稳定甚至NaN梯度。建议先用start_pref_f=100，limit_pref_f=60，其余loss参数保持不变，参考官方DPA3参数表。
核查descriptor参数：e_sel和a_sel建议与官方示例保持一致（如120/30或154/48），你的e_sel=1200、a_sel=300远大于推荐值，可能导致内存或数值问题。建议先用120/30或154/48测试。
确认分支代码修复：DPA3模型在2025年有重要修复（PR fix(jax): fix Hessian NaN for DPA-3 #4668、fix(jax): fix DPA3 force NaN with edge_init_use_dist #4794），用safe_for_vector_norm替换了原始的vector_norm，防止NaN梯度导致训练崩溃。如果你的Alloy_tongqi分支没有合并这些修复，建议合并或升级到包含这些修复的版本，详见PR #4668 和 PR #4794。
最小化测试：可用一小部分已验证无异常的数据和官方推荐参数做一次短训练，若能正常训练，说明问题主要在数据或参数设置。
数据质量核查：确保所有训练数据单位、符号一致，能量梯度为负的力，SCF收敛，去除不稳定的异常点，数据精度足够（建议double精度存储）。
stat_file问题：如果stat_file被损坏或包含NaN/Inf，也会导致训练异常。可以尝试删除stat_file让程序重新生成。

优先建议：先检查数据和stat_file有无NaN/Inf，调整loss权重和descriptor参数到官方推荐范围，再确认分支代码是否包含上述数值稳定性修复。这样可大幅降低训练中出现NaN梯度的概率。官方DPA3参数表和说明

To reply, just mention @dosu.

^{How did I do? Good | Irrelevant | Incorrect | Verbose | Hallucination | Report 🐛 | Other}

1 reply

iProzd Jan 22, 2026
Maintainer

Hi @Zch102xjtumse , as the AI suggested, this might probably result from your dataset.
To check this, you can:

Check if there's unusual data point(NaN labels or structures with over-close atoms) .
Instead of fine-tuning, check if training from scratch works well.

And besides these checks, could you also provide the lcurve.out so that we can see when did this happen.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

DPA3 finetune Error #5169

Uh oh!

{{title}}

Uh oh!

Replies: 1 comment 1 reply

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

DPA3 finetune Error #5169

Uh oh!

Zch102xjtumse Jan 21, 2026

Replies: 1 comment · 1 reply

Uh oh!

dosubot[bot] bot Jan 21, 2026

Uh oh!

iProzd Jan 22, 2026 Maintainer

Zch102xjtumse
Jan 21, 2026

Replies: 1 comment 1 reply

dosubot[bot]
bot Jan 21, 2026

iProzd Jan 22, 2026
Maintainer