@@ -49,52 +49,86 @@ def daterange(date1, date2):
4949 return dates
5050
5151
52- def generate_index (log_dir = "log" ):
53- """generates index.json with dates <- [coins] """
52+ def gather_symbols_and_logs (log_dir = "log" ) -> tuple [ set [ str ], set [ str ]] :
53+ """returns lists of symbols and dates """
5454 date_list = set ()
5555 symbols_list = set ()
56- index = {}
5756
5857 # gather all date.log.gz logs and
5958 # all symbol dirs
60- for item in sorted (os .listdir (log_dir )):
59+ for dir_item in sorted (os .listdir (log_dir )):
6160 if (
62- os .path .isfile (f"{ log_dir } { item } )
63- and item .startswith ("20" )
64- and ".log." in item
61+ os .path .isfile (f"{ log_dir } { dir_item } )
62+ and dir_item .startswith ("20" )
63+ and dir_item . endswith ( ".log.gz" )
6564 ):
66- date = item .split ("." )[0 ]
65+ date : str = dir_item .split ("." )[0 ]
6766 date_list .add (date )
68- if os .path .isdir (f"{ log_dir } { item } ):
69- symbols_list .add (item )
67+ if os .path .isdir (f"{ log_dir } { dir_item } ):
68+ symbols_list .add (dir_item )
69+
70+ return (set (symbols_list ), set (date_list ))
71+
72+
73+ def gather_symbols_per_date (
74+ log_dir , symbols_list , date_list
75+ ) -> dict [str , list [str ]]:
76+ """returns map of dates containing symbols available on that date"""
77+ dates_idx : dict [str , list [str ]] = {}
7078
7179 # we'll store all symbol logs in each date
7280 for date in sorted (date_list ):
73- index [date ] = set ()
81+ if date not in dates_idx :
82+ dates_idx [date ] = []
7483
75- # iterate over all the symbols and gather all the
76- # logfiles in in each one of those symbol dirs
7784 for _symbol in sorted (symbols_list ):
78- logs = os .listdir (f"{ log_dir } { _symbol } )
85+ logs : list [ str ] = os .listdir (f"{ log_dir } { _symbol } )
7986 for _log in sorted (logs ):
8087 if not os .path .isfile (f"{ log_dir } { _symbol } { _log } ):
8188 continue
82- date = _log .split ("." )[0 ]
83- index [date ].add (_symbol )
89+ _date : str = _log .split ("." )[0 ]
90+ dates_idx [_date ].append (_symbol )
91+ return dates_idx
8492
85- tmp = index
86- index = {}
87- for date in tmp .keys (): # pylint: disable=C0206,C0201
88- index [date ] = list (tmp [date ])
93+
94+ def generate_index (log_dir = "log" ) -> None :
95+ """generates index.json with dates <- [coins]"""
96+
97+ print ("generating index..." )
98+ symbols_list , date_list = gather_symbols_and_logs (log_dir )
99+
100+ dates_index : dict [str , list [str ]] = gather_symbols_per_date (
101+ log_dir , symbols_list , date_list
102+ )
103+
104+ # generate index_v1
105+ print ("writing index.json.gz..." )
89106
90107 with gzip .open (
91108 f"{ log_dir } , "wt" , encoding = "utf-8"
109+ ) as index_json :
110+ index_json .write (json .dumps (dates_index , indent = 4 ))
111+
112+ # generate index_v2
113+ print ("generating index_v2.json.gz..." )
114+ index : dict [str , dict ] = {"DATES" : {}, "COINS" : {}}
115+ for date in dates_index .keys (): # pylint: disable=C0206,C0201
116+ index ["DATES" ][date ] = list (dates_index [date ])
117+
118+ for _symbol in sorted (os .listdir (log_dir )):
119+ if os .path .isdir (f"{ log_dir } { _symbol } ):
120+ logs : list [str ] = os .listdir (f"{ log_dir } { _symbol } )
121+ index ["COINS" ][_symbol ] = sorted (logs )
122+
123+ print ("writing index_v2.json.gz..." )
124+ with gzip .open (
125+ f"{ log_dir } , "wt" , encoding = "utf-8"
92126 ) as index_json :
93127 index_json .write (json .dumps (index , indent = 4 ))
94128
95129
96130if __name__ == "__main__" :
97- parser = argparse .ArgumentParser ()
131+ parser : argparse . ArgumentParser = argparse .ArgumentParser ()
98132 parser .add_argument ("-s" , "--start" , help = "start day to fetch klines for" )
99133 parser .add_argument (
100134 "-e" , "--end" , help = "end day to fetch klines for" , required = False
0 commit comments