update remaining issues and added benchmark results.

README.md

@@ -294,11 +294,10 @@ You can find a [table](training/results/README.md) of common benchmark datasets
 # Issues
 
 Some known issues to be aware of, if using and making new models or layers with `kgcnn`.
-* Jagged or nested Tensors loading into models for PyTorch backend is not working.  
-* Dataloader for Jax is implemented but does not provide all functionalities like output padding or dictionary inputs.
-* ForceModel does not support all tensor types and does not have Scaler layer yet.
+* Jagged or nested Tensors loading into models for PyTorch backend is not working.
+* ForceModel does not support all backends and does not have Scaler layer yet.
 * BatchNormalization layer dos not support padding yet.
-
+* Keras AUC metric does not seem to work for torch cuda.
 
 <a name="citing"></a>
 # Citing

training/results/MatProjectLogGVRHDataset/DimeNetPP_make_crystal_model/DimeNetPP_MatProjectLogGVRHDataset_score.yaml

@@ -0,0 +1,138 @@
+OS: posix_linux
+backend: tensorflow
+cuda_available: 'True'
+data_unit: GPa
+date_time: '2024-01-08 06:45:52'
+device_id: '[LogicalDevice(name=''/device:CPU:0'', device_type=''CPU''), LogicalDevice(name=''/device:GPU:0'',
+  device_type=''GPU'')]'
+device_memory: '[]'
+device_name: '[{}, {''compute_capability'': (8, 0), ''device_name'': ''NVIDIA A100
+  80GB PCIe''}]'
+epochs:
+- 780
+- 780
+- 780
+- 780
+- 780
+execute_folds: null
+kgcnn_version: 4.0.0
+loss:
+- 0.010922136716544628
+- 0.012440935708582401
+- 0.011790135875344276
+- 0.012489846907556057
+- 0.010465575382113457
+max_loss:
+- 0.49855291843414307
+- 0.4948725402355194
+- 0.48957857489585876
+- 0.49504366517066956
+- 0.4858188033103943
+max_scaled_mean_absolute_error:
+- 0.18419674038887024
+- 0.18402329087257385
+- 0.18198589980602264
+- 0.1838250607252121
+- 0.17966441810131073
+max_scaled_root_mean_squared_error:
+- 0.26791346073150635
+- 0.2653201222419739
+- 0.26441052556037903
+- 0.26360654830932617
+- 0.2580696642398834
+max_val_loss:
+- 0.2740524113178253
+- 0.2666001319885254
+- 0.2633107006549835
+- 0.2688142955303192
+- 0.2762555480003357
+max_val_scaled_mean_absolute_error:
+- 0.10124088823795319
+- 0.09903348237276077
+- 0.09774773567914963
+- 0.09964113682508469
+- 0.10210282355546951
+max_val_scaled_root_mean_squared_error:
+- 0.14148077368736267
+- 0.14824888110160828
+- 0.14390243589878082
+- 0.14439791440963745
+- 0.14645302295684814
+min_loss:
+- 0.010105308145284653
+- 0.011313959024846554
+- 0.01063369121402502
+- 0.011703375726938248
+- 0.009394158609211445
+min_scaled_mean_absolute_error:
+- 0.003735009813681245
+- 0.004206336103379726
+- 0.003950342070311308
+- 0.004348854534327984
+- 0.0034740078262984753
+min_scaled_root_mean_squared_error:
+- 0.007411751430481672
+- 0.007685335818678141
+- 0.006682842504233122
+- 0.007493346463888884
+- 0.0061401729471981525
+min_val_loss:
+- 0.2148033231496811
+- 0.2151896208524704
+- 0.21606729924678802
+- 0.23161034286022186
+- 0.21440891921520233
+min_val_scaled_mean_absolute_error:
+- 0.07936041802167892
+- 0.07992357015609741
+- 0.08021173626184464
+- 0.08571653813123703
+- 0.07917997241020203
+min_val_scaled_root_mean_squared_error:
+- 0.1176394373178482
+- 0.1263747215270996
+- 0.12576667964458466
+- 0.1290467530488968
+- 0.11715161800384521
+model_class: make_crystal_model
+model_name: DimeNetPP
+model_version: '2023-12-04'
+multi_target_indices: null
+number_histories: 5
+scaled_mean_absolute_error:
+- 0.004034512676298618
+- 0.004626349080353975
+- 0.004383227322250605
+- 0.00463763065636158
+- 0.003870107466354966
+scaled_root_mean_squared_error:
+- 0.008117007091641426
+- 0.008247202262282372
+- 0.00719984108582139
+- 0.00837814249098301
+- 0.006836302578449249
+seed: 42
+time_list:
+- '6:13:20.600089'
+- '6:09:44.598682'
+- '6:40:04.415875'
+- '6:53:14.904282'
+- '6:42:50.116184'
+val_loss:
+- 0.22598953545093536
+- 0.22380274534225464
+- 0.2243814319372177
+- 0.2401961386203766
+- 0.2184777557849884
+val_scaled_mean_absolute_error:
+- 0.08350197225809097
+- 0.08309303969144821
+- 0.0832805335521698
+- 0.0888705849647522
+- 0.08074352145195007
+val_scaled_root_mean_squared_error:
+- 0.12264644354581833
+- 0.13055872917175293
+- 0.1287512332201004
+- 0.1403961032629013
+- 0.1228041797876358

training/results/MatProjectLogGVRHDataset/DimeNetPP_make_crystal_model/DimeNetPP_hyper.json

@@ -0,0 +1 @@
+{"model": {"class_name": "make_crystal_model", "module_name": "kgcnn.literature.DimeNetPP", "config": {"name": "DimeNetPP", "inputs": [{"shape": [null], "name": "node_number", "dtype": "int32", "ragged": true}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32", "ragged": true}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64", "ragged": true}, {"shape": [null, 2], "name": "angle_indices", "dtype": "int64", "ragged": true}, {"shape": [null, 3], "name": "range_image", "dtype": "int64", "ragged": true}, {"shape": [3, 3], "name": "graph_lattice", "dtype": "float32", "ragged": false}], "input_tensor_type": "ragged", "input_embedding": null, "input_node_embedding": {"input_dim": 95, "output_dim": 128, "embeddings_initializer": {"class_name": "RandomUniform", "config": {"minval": -1.7320508075688772, "maxval": 1.7320508075688772}}}, "emb_size": 128, "out_emb_size": 256, "int_emb_size": 64, "basis_emb_size": 8, "num_blocks": 4, "num_spherical": 7, "num_radial": 6, "cutoff": 5.0, "envelope_exponent": 5, "num_before_skip": 1, "num_after_skip": 2, "num_dense_output": 3, "num_targets": 1, "extensive": false, "output_init": "zeros", "activation": "swish", "verbose": 10, "output_embedding": "graph", "use_output_mlp": false, "output_mlp": {}}}, "training": {"cross_validation": {"class_name": "KFold", "config": {"n_splits": 5, "random_state": 42, "shuffle": true}}, "fit": {"batch_size": 16, "epochs": 780, "validation_freq": 10, "verbose": 2, "callbacks": [], "validation_batch_size": 8}, "compile": {"optimizer": {"class_name": "Adam", "config": {"learning_rate": {"class_name": "kgcnn>LinearWarmupExponentialDecay", "config": {"learning_rate": 0.001, "warmup_steps": 3000.0, "decay_steps": 4000000.0, "decay_rate": 0.01}}, "use_ema": true, "amsgrad": true}}, "loss": "mean_absolute_error"}, "scaler": {"class_name": "StandardLabelScaler", "module_name": "kgcnn.data.transform.scaler.standard", "config": {"with_std": true, "with_mean": true, "copy": true}}, "multi_target_indices": null}, "data": {}, "info": {"postfix": "", "postfix_file": "", "kgcnn_version": "4.0.0"}, "dataset": {"class_name": "MatProjectLogGVRHDataset", "module_name": "kgcnn.data.datasets.MatProjectLogGVRHDataset", "config": {}, "methods": [{"map_list": {"method": "set_range_periodic", "max_distance": 5.0, "max_neighbours": 17}}, {"map_list": {"method": "set_angle", "allow_multi_edges": true, "allow_reverse_edges": true}}]}}

training/results/MatProjectLogGVRHDataset/Megnet_make_crystal_model/Megnet_MatProjectLogGVRHDataset_score.yaml

@@ -0,0 +1,156 @@
+OS: posix_linux
+backend: tensorflow
+cuda_available: 'True'
+data_unit: GPa
+date_time: '2024-01-04 12:06:59'
+device_id: '[LogicalDevice(name=''/device:CPU:0'', device_type=''CPU''), LogicalDevice(name=''/device:GPU:0'',
+  device_type=''GPU'')]'
+device_memory: '[]'
+device_name: '[{}, {''compute_capability'': (8, 0), ''device_name'': ''NVIDIA A100
+  80GB PCIe''}]'
+epochs:
+- 1000
+- 1000
+- 1000
+- 1000
+- 1000
+execute_folds: null
+kgcnn_version: 4.0.0
+learning_rate:
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+loss:
+- 0.008964268490672112
+- 0.01045008935034275
+- 0.009522628970444202
+- 0.009219203144311905
+- 0.009494253434240818
+max_learning_rate:
+- 0.0005000000237487257
+- 0.0005000000237487257
+- 0.0005000000237487257
+- 0.0005000000237487257
+- 0.0005000000237487257
+max_loss:
+- 0.4917912185192108
+- 0.4792233109474182
+- 0.4805443286895752
+- 0.4773976802825928
+- 0.48819148540496826
+max_scaled_mean_absolute_error:
+- 0.18157169222831726
+- 0.1780826598405838
+- 0.17843185365200043
+- 0.1771635264158249
+- 0.18046769499778748
+max_scaled_root_mean_squared_error:
+- 0.25538089871406555
+- 0.25090330839157104
+- 0.2512081563472748
+- 0.24878676235675812
+- 0.25342148542404175
+max_val_loss:
+- 0.3892642557621002
+- 0.3616122603416443
+- 0.36633700132369995
+- 0.3724639117717743
+- 0.3718774914741516
+max_val_scaled_mean_absolute_error:
+- 0.1435098499059677
+- 0.1340455263853073
+- 0.13566848635673523
+- 0.1377905160188675
+- 0.13669702410697937
+max_val_scaled_root_mean_squared_error:
+- 0.20212596654891968
+- 0.19816583395004272
+- 0.18757493793964386
+- 0.20138297975063324
+- 0.1989428699016571
+min_learning_rate:
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+- 5.549999968934571e-06
+min_loss:
+- 0.00761491060256958
+- 0.010199162177741528
+- 0.009248618967831135
+- 0.009219203144311905
+- 0.00822273176163435
+min_scaled_mean_absolute_error:
+- 0.002814092906191945
+- 0.0037900579627603292
+- 0.0034343195147812366
+- 0.003426043316721916
+- 0.00304029299877584
+min_scaled_root_mean_squared_error:
+- 0.006396596785634756
+- 0.008159836754202843
+- 0.008076793514192104
+- 0.006940171588212252
+- 0.008059120737016201
+min_val_loss:
+- 0.2344822883605957
+- 0.2433883547782898
+- 0.23295150697231293
+- 0.24354171752929688
+- 0.24186797440052032
+min_val_scaled_mean_absolute_error:
+- 0.08650889247655869
+- 0.09005645662546158
+- 0.0863751545548439
+- 0.08979988098144531
+- 0.08870527148246765
+min_val_scaled_root_mean_squared_error:
+- 0.12720821797847748
+- 0.13807186484336853
+- 0.13143642246723175
+- 0.1411106437444687
+- 0.1333688497543335
+model_class: make_crystal_model
+model_name: Megnet
+model_version: '2023-12-05'
+multi_target_indices: null
+number_histories: 5
+scaled_mean_absolute_error:
+- 0.0033130773808807135
+- 0.0038830169942229986
+- 0.0035354492720216513
+- 0.003426043316721916
+- 0.0035096919164061546
+scaled_root_mean_squared_error:
+- 0.0068201362155377865
+- 0.008816206827759743
+- 0.008103175088763237
+- 0.007409997750073671
+- 0.008422690443694592
+seed: 42
+time_list:
+- '2:32:49.335213'
+- '2:34:01.461139'
+- '2:39:41.452388'
+- '2:40:30.004558'
+- '2:42:00.905719'
+val_loss:
+- 0.23461347818374634
+- 0.24520882964134216
+- 0.23295150697231293
+- 0.24354171752929688
+- 0.24289974570274353
+val_scaled_mean_absolute_error:
+- 0.08654788881540298
+- 0.09072180092334747
+- 0.0863751545548439
+- 0.08979988098144531
+- 0.08907058835029602
+val_scaled_root_mean_squared_error:
+- 0.1280117928981781
+- 0.14162039756774902
+- 0.13248546421527863
+- 0.1423494815826416
+- 0.13548214733600616

training/results/MatProjectLogGVRHDataset/Megnet_make_crystal_model/Megnet_hyper.json

@@ -0,0 +1 @@
+{"model": {"module_name": "kgcnn.literature.Megnet", "class_name": "make_crystal_model", "config": {"name": "Megnet", "inputs": [{"shape": [null], "name": "node_number", "dtype": "int32", "ragged": true}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32", "ragged": true}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64", "ragged": true}, {"shape": [1], "name": "charge", "dtype": "float32", "ragged": false}, {"shape": [null, 3], "name": "range_image", "dtype": "int64", "ragged": true}, {"shape": [3, 3], "name": "graph_lattice", "dtype": "float32", "ragged": false}], "input_tensor_type": "ragged", "input_embedding": null, "input_node_embedding": {"input_dim": 95, "output_dim": 64}, "make_distance": true, "expand_distance": true, "gauss_args": {"bins": 25, "distance": 5, "offset": 0.0, "sigma": 0.4}, "meg_block_args": {"node_embed": [64, 32, 32], "edge_embed": [64, 32, 32], "env_embed": [64, 32, 32], "activation": "kgcnn>softplus2"}, "set2set_args": {"channels": 16, "T": 3, "pooling_method": "sum", "init_qstar": "0"}, "node_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"}, "edge_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"}, "state_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"}, "nblocks": 3, "has_ff": true, "dropout": null, "use_set2set": true, "verbose": 10, "output_embedding": "graph", "output_mlp": {"use_bias": [true, true, true], "units": [32, 16, 1], "activation": ["kgcnn>softplus2", "kgcnn>softplus2", "linear"]}}}, "training": {"cross_validation": {"class_name": "KFold", "config": {"n_splits": 5, "random_state": 42, "shuffle": true}}, "fit": {"batch_size": 32, "epochs": 1000, "validation_freq": 10, "verbose": 2, "callbacks": [{"class_name": "kgcnn>LinearLearningRateScheduler", "config": {"learning_rate_start": 0.0005, "learning_rate_stop": 5e-06, "epo_min": 100, "epo": 1000, "verbose": 0}}]}, "compile": {"optimizer": {"class_name": "Adam", "config": {"learning_rate": 0.0005}}, "loss": "mean_absolute_error"}, "scaler": {"class_name": "StandardLabelScaler", "module_name": "kgcnn.data.transform.scaler.standard", "config": {"with_std": true, "with_mean": true, "copy": true}}, "multi_target_indices": null}, "data": {"data_unit": "GPa"}, "info": {"postfix": "", "postfix_file": "", "kgcnn_version": "4.0.0"}, "dataset": {"class_name": "MatProjectLogGVRHDataset", "module_name": "kgcnn.data.datasets.MatProjectLogGVRHDataset", "config": {}, "methods": [{"map_list": {"method": "set_range_periodic", "max_distance": 5.0}}]}}