Skip to content

Commit

Permalink
update training results. Added hyperparameters.
Browse files Browse the repository at this point in the history
  • Loading branch information
@PatReis
PatReis committed Jan 2, 2024
1 parent be67e70 commit 0c20ad5
Show file tree
Hide file tree
Showing 10 changed files with 1,751 additions and 37 deletions.
3 changes: 2 additions & 1 deletion changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Completely reworked version of kgcnn for Keras 3.0 and multi-backend support. A
However, we tried to keep as much of the API from kgcnn 3.0 so that models in literature can be used with minimal changes.
Mainly, the ``"input_tensor_type"="ragged"`` model parameter has to be added if ragged tensors are used as input in tensorflow.
For very few models also the order of inputs had to be changed.
Also note that the input embedding layer requires integer tensor input and does not cast from float anymore.

The scope of models has been reduced for initial release but will be extended in upcoming versions.
Note that some changes are also stem for keras API changes, like for example *learning_rate* rate parameter or serialization.
Expand All @@ -15,7 +16,7 @@ Input can be padded or (still) ragged input. Or direct disjoint representations
(See models chapter in docs).

For jax we added a ``padded_disjoint`` parameter that can enable jit'able jax models but requires a dataloader,
which is not yet implemented in ``kgcnn``. For padded samples it can already been tested,
which is not yet thoroughly implemented in ``kgcnn`` . For padded samples it can already been tested,
but the padding of each sample is a much larger overhead than padding the batch.

Some other changes:
Expand Down
237 changes: 232 additions & 5 deletions training/hyper/hyper_mp_e_form.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,15 @@
"config": {
"name": "PAiNN",
"inputs": [
{"shape": [None], "name": "node_number", "dtype": "float32", "ragged": True},
{"shape": [None], "name": "node_number", "dtype": "int32", "ragged": True},
{"shape": [None, 3], "name": "node_coordinates", "dtype": "float32", "ragged": True},
{"shape": [None, 2], "name": "range_indices", "dtype": "int64", "ragged": True},
{'shape': (None, 3), 'name': "range_image", 'dtype': 'int64', 'ragged': True},
{'shape': (3, 3), 'name': "graph_lattice", 'dtype': 'float32', 'ragged': False}
],
"input_embedding": {"node": {"input_dim": 95, "output_dim": 128}},
"input_tensor_type": "ragged",
"input_embedding": None,
"input_node_embedding": {"input_dim": 95, "output_dim": 128},
"bessel_basis": {"num_radial": 20, "cutoff": 5.0, "envelope_exponent": 5},
"equiv_initialize_kwargs": {"dim": 3, "method": "eye"},
"pooling_args": {"pooling_method": "mean"},
Expand All @@ -109,11 +111,84 @@
]
},
"compile": {
"optimizer": {"class_name": "Adam", "config": {"lr": 0.0001}},
"optimizer": {"class_name": "Adam", "config": {"learning_rate": 0.0001}},
"loss": "mean_absolute_error"
},
"scaler": {
"class_name": "StandardScaler",
"class_name": "StandardLabelScaler",
"module_name": "kgcnn.data.transform.scaler.standard",
"config": {"with_std": True, "with_mean": True, "copy": True}
},
"multi_target_indices": None
},
"data": {
"dataset": {
"class_name": "MatProjectEFormDataset",
"module_name": "kgcnn.data.datasets.MatProjectEFormDataset",
"config": {},
"methods": [
{"map_list": {"method": "set_range_periodic", "max_distance": 5.0}}
]
},
"data_unit": "eV/atom"
},
"info": {
"postfix": "",
"postfix_file": "",
"kgcnn_version": "4.0.0"
}
},
"Megnet.make_crystal_model": {
"model": {
"module_name": "kgcnn.literature.Megnet",
"class_name": "make_crystal_model",
"config": {
'name': "Megnet",
'inputs': [
{'shape': (None,), 'name': "node_number", 'dtype': 'int32', 'ragged': True},
{'shape': (None, 3), 'name': "node_coordinates", 'dtype': 'float32', 'ragged': True},
{'shape': (None, 2), 'name': "range_indices", 'dtype': 'int64', 'ragged': True},
{'shape': [1], 'name': "charge", 'dtype': 'float32', 'ragged': False},
{'shape': (None, 3), 'name': "range_image", 'dtype': 'int64', 'ragged': True},
{'shape': (3, 3), 'name': "graph_lattice", 'dtype': 'float32', 'ragged': False}
],
"input_tensor_type": "ragged",
'input_embedding': None,
"input_node_embedding": {"input_dim": 95, "output_dim": 64},
# "input_edge_embedding": {"input_dim": 100, "output_dim": 64},
"make_distance": True, "expand_distance": True,
'gauss_args': {"bins": 25, "distance": 5, "offset": 0.0, "sigma": 0.4},
'meg_block_args': {'node_embed': [64, 32, 32], 'edge_embed': [64, 32, 32],
'env_embed': [64, 32, 32], 'activation': 'kgcnn>softplus2'},
'set2set_args': {'channels': 16, 'T': 3, "pooling_method": "sum", "init_qstar": "0"},
'node_ff_args': {"units": [64, 32], "activation": "kgcnn>softplus2"},
'edge_ff_args': {"units": [64, 32], "activation": "kgcnn>softplus2"},
'state_ff_args': {"units": [64, 32], "activation": "kgcnn>softplus2"},
'nblocks': 3, 'has_ff': True, 'dropout': None, 'use_set2set': True,
'verbose': 10,
'output_embedding': 'graph',
'output_mlp': {"use_bias": [True, True, True], "units": [32, 16, 1],
"activation": ['kgcnn>softplus2', 'kgcnn>softplus2', 'linear']}
}
},
"training": {
"cross_validation": {"class_name": "KFold",
"config": {"n_splits": 5, "random_state": 42, "shuffle": True}},
"fit": {
"batch_size": 32, "epochs": 1000, "validation_freq": 10, "verbose": 2,
"callbacks": [
{"class_name": "kgcnn>LinearLearningRateScheduler", "config": {
"learning_rate_start": 0.0005, "learning_rate_stop": 0.5e-05, "epo_min": 100, "epo": 1000,
"verbose": 0}
}
]
},
"compile": {
"optimizer": {"class_name": "Adam", "config": {"learning_rate": 0.0005}},
"loss": "mean_absolute_error"
},
"scaler": {
"class_name": "StandardLabelScaler",
"module_name": "kgcnn.data.transform.scaler.standard",
"config": {"with_std": True, "with_mean": True, "copy": True}
},
Expand All @@ -133,7 +208,159 @@
"info": {
"postfix": "",
"postfix_file": "",
"kgcnn_version": "2.0.4"
"kgcnn_version": "4.0.0"
}
},
"DimeNetPP.make_crystal_model": {
"model": {
"class_name": "make_crystal_model",
"module_name": "kgcnn.literature.DimeNetPP",
"config": {
"name": "DimeNetPP",
"inputs": [
{"shape": [None], "name": "node_number", "dtype": "int32", "ragged": True},
{"shape": [None, 3], "name": "node_coordinates", "dtype": "float32", "ragged": True},
{"shape": [None, 2], "name": "range_indices", "dtype": "int64", "ragged": True},
{"shape": [None, 2], "name": "angle_indices", "dtype": "int64", "ragged": True},
{'shape': (None, 3), 'name': "range_image", 'dtype': 'int64', 'ragged': True},
{'shape': (3, 3), 'name': "graph_lattice", 'dtype': 'float32', 'ragged': False}
],
"input_tensor_type": "ragged",
"input_embedding": None,
"input_node_embedding": {"input_dim": 95, "output_dim": 128,
"embeddings_initializer": {"class_name": "RandomUniform",
"config": {"minval": -1.7320508075688772,
"maxval": 1.7320508075688772}}},
"emb_size": 128, "out_emb_size": 256, "int_emb_size": 64, "basis_emb_size": 8,
"num_blocks": 4, "num_spherical": 7, "num_radial": 6,
"cutoff": 5.0, "envelope_exponent": 5,
"num_before_skip": 1, "num_after_skip": 2, "num_dense_output": 3,
"num_targets": 1, "extensive": False, "output_init": "zeros",
"activation": "swish", "verbose": 10,
"output_embedding": "graph",
"use_output_mlp": False,
"output_mlp": {},
}
},
"training": {
"cross_validation": {"class_name": "KFold",
"config": {"n_splits": 5, "random_state": 42, "shuffle": True}},
"fit": {
"batch_size": 16, "epochs": 780, "validation_freq": 10, "verbose": 2, "callbacks": [],
"validation_batch_size": 8
},
"compile": {
"optimizer": {
"class_name": "Adam",
"config": {
"learning_rate": {
"class_name": "kgcnn>LinearWarmupExponentialDecay", "config": {
"learning_rate": 0.001, "warmup_steps": 3000.0, "decay_steps": 4000000.0,
"decay_rate": 0.01
}
},
"use_ema": True,
"amsgrad": True,
}
},
"loss": "mean_absolute_error"
},
"scaler": {
"class_name": "StandardLabelScaler",
"module_name": "kgcnn.data.transform.scaler.standard",
"config": {"with_std": True, "with_mean": True, "copy": True}
},
"multi_target_indices": None
},
"data": {
"dataset": {
"class_name": "MatProjectEFormDataset",
"module_name": "kgcnn.data.datasets.MatProjectEFormDataset",
"config": {},
"methods": [
{"map_list": {"method": "set_range_periodic", "max_distance": 5.0, "max_neighbours": 17}},
{"map_list": {"method": "set_angle", "allow_multi_edges": True, "allow_reverse_edges": True}}
]
},
},
"info": {
"postfix": "",
"postfix_file": "",
"kgcnn_version": "4.0.0"
}
},
"CGCNN.make_crystal_model": {
"model": {
"class_name": "make_crystal_model",
"module_name": "kgcnn.literature.CGCNN",
"config": {
'name': 'CGCNN',
'inputs': [
{'shape': (None,), 'name': 'node_number', 'dtype': 'int64', 'ragged': True},
{'shape': (None, 3), 'name': 'node_frac_coordinates', 'dtype': 'float64', 'ragged': True},
{'shape': (None, 2), 'name': 'range_indices', 'dtype': 'int64', 'ragged': True},
{'shape': (None, 3), 'name': 'range_image', 'dtype': 'float32', 'ragged': True},
{'shape': (3, 3), 'name': 'graph_lattice', 'dtype': 'float64', 'ragged': False},
# For `representation="asu"`:
# {'shape': (None, 1), 'name': 'multiplicities', 'dtype': 'float32', 'ragged': True},
# {'shape': (None, 4, 4), 'name': 'symmops', 'dtype': 'float64', 'ragged': True},
],
"input_tensor_type": "ragged",
'input_node_embedding': {'input_dim': 95, 'output_dim': 64},
'representation': 'unit', # None, 'asu' or 'unit'
'expand_distance': True,
'make_distances': True,
'gauss_args': {'bins': 60, 'distance': 6, 'offset': 0.0, 'sigma': 0.4},
'conv_layer_args': {
'units': 128,
'activation_s': 'kgcnn>shifted_softplus',
'activation_out': 'kgcnn>shifted_softplus',
'batch_normalization': True,
},
'node_pooling_args': {'pooling_method': 'mean'},
'depth': 4,
'output_mlp': {'use_bias': [True, True, False], 'units': [128, 64, 1],
'activation': ['kgcnn>shifted_softplus', 'kgcnn>shifted_softplus', 'linear']},
}
},
"training": {
"cross_validation": {"class_name": "KFold",
"config": {"n_splits": 5, "random_state": 42, "shuffle": True}},
"fit": {
"batch_size": 128, "epochs": 1000, "validation_freq": 10, "verbose": 2,
"callbacks": [
{"class_name": "kgcnn>LinearLearningRateScheduler", "config": {
"learning_rate_start": 1e-03, "learning_rate_stop": 1e-05, "epo_min": 500, "epo": 1000,
"verbose": 0}
}
]
},
"compile": {
"optimizer": {"class_name": "Adam", "config": {"learning_rate": 1e-03}},
"loss": "mean_absolute_error"
},
"scaler": {
"class_name": "StandardLabelScaler",
"module_name": "kgcnn.data.transform.scaler.standard",
"config": {"with_std": True, "with_mean": True, "copy": True}
},
"multi_target_indices": None
},
"data": {
"dataset": {
"class_name": "MatProjectEFormDataset",
"module_name": "kgcnn.data.datasets.MatProjectEFormDataset",
"config": {},
"methods": [
{"map_list": {"method": "set_range_periodic", "max_distance": 6.0}}
]
},
"data_unit": "eV/atom"
},
"info": {
"postfix": "",
"postfix_file": "",
"kgcnn_version": "4.0.0"
}
},
}
Loading

0 comments on commit 0c20ad5

Please sign in to comment.