-
Notifications
You must be signed in to change notification settings - Fork 0
/
ModelCardGenerator.py
84 lines (68 loc) · 4.16 KB
/
ModelCardGenerator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
import cmd
class ModelCardGenerator(cmd.Cmd):
intro = " Welcome to the model card generator developed by xD|Census!\n\n This tool can be used to create a model card: a way to share information about a model's intent, data, architecture, and performance.\n\n To generate a markdown (*.md) file version of a model card, type \"begin\", hit enter, and enter the model\'s information as prompted.\n\n To exit the script at any time without saving the file, type \"exit\" and hit enter."
prompt = '> '
file = None
model_name = None
statements_dict = {
'Accountability': ['Model name', 'Point of contact(s)', 'Agencies/Affiliations' 'Model acquisition/development method'],
'Anticipated Use': ['Division(s) using the model', 'Intended application(s)', 'Intended stakeholder(s)'],
'Model Information & Architecture': ['Current model version', 'Version release date', 'Changes made since the last release (if any)', 'License for use', 'Type of model (Classification, Regression, Object Detection, etc.)', 'Type of algorithm(s) used'],
'Dataset & Performance': ['Source(s) of the training data', 'Data collection/ generation method', 'Number of variables in this dataset', 'Number of entries in your dataset', 'Percent of data chosen as a training, testing and validation sets', 'Metrics used to rate model performance', 'Factors that limit the model\'s performance. (Examples: Limited dataset, number of nulls/NAs) (if any)'],
'Bias Identification & Mitigation': ['Inclusion of information related to individuals or human populations in the training/testing/validation datasets - Yes/No, if "yes" include a description of demographic features', 'Methods used to minimize bias from human judgement', 'Potential biases found in the training dataset from collection methods, sample size, representation, etc.', 'Testing/evaluation performed to look for bias in the workflow of the model', 'Degree of model explainability/transparency'],
'Governance/Compliance': ['Model/dataset compliance with existing laws and regulations. (Including privacy protection regulations)']
}
def do_begin(self, arg):
for key in self.statements_dict:
# first section: initialize file
if key == 'Accountability':
self.accountability(arg)
# last section: save file and exit
elif key == 'Governance/Compliance':
self.complete_file(arg)
else:
self.write_section(key)
def write_section(self, section):
self.file.write(f"## {section}\n")
for segment in self.statements_dict[section]:
input_val = input(f"{segment}\n")
self.file.write(f"### {segment}\n")
self.file.write(f"* {input_val}\n")
def accountability(self, _arg):
for section in self.statements_dict['Accountability']:
if section == 'Model name':
model_name = input(f"{section}\n")
self.init_model_name(model_name)
else:
input_val = input(f"{section}\n")
self.file.write(f"### {section}\n")
self.file.write(f"* {input_val}\n")
def init_model_name(self, name):
self.model_name = name
self.init_file()
def init_file(self):
filename = self.model_name
self.file = open(f"{filename}.md", 'w')
self.file.write(f"# {self.model_name} Model Card\n")
self.file.write("## Accountability\n")
self.file.write("### Model Name\n")
self.file.write(f"* {self.model_name}\n")
def complete_file(self, _arg):
section = 'Governance/Compliance'
self.file.write(f"## {section}\n")
for segment in self.statements_dict[section]:
input_val = input(f"{segment}\n")
self.file.write(f"### {segment}\n")
self.file.write(f"* {input_val}\n")
self.close()
exit()
def do_exit(self, _arg):
'Stop recording, close the tool, and exit'
self.close()
exit()
def close(self):
if self.file:
self.file.close()
self.file = None
if __name__ == '__main__':
ModelCardGenerator().cmdloop()