diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index fe2432553..9d86a737c 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -65,6 +65,8 @@ jobs: - name: DB Initialization run: | docker exec anyway alembic upgrade head + docker exec anyway ./main.py process cities + docker exec anyway ./main.py process streets docker exec anyway ./main.py process registered-vehicles docker exec anyway ./main.py process cbs --source local_dir_for_tests_only docker exec anyway ./main.py process road-segments diff --git a/anyway/localization.py b/anyway/localization.py index aee40b83c..7a41ce11e 100644 --- a/anyway/localization.py +++ b/anyway/localization.py @@ -3,7 +3,6 @@ import pandas as pd from anyway import field_names -from typing import Optional import logging @@ -214,11 +213,3 @@ def get_field(field, value=None): def get_supported_tables(): return _tables.keys() - - -def get_city_name(symbol_id, lang: str = "he") -> Optional[str]: - column_to_fetch = field_names.name if lang == "he" else "ENGLISH_NAME" - try: - return _cities.loc[symbol_id, column_to_fetch] - except Exception: - return None diff --git a/anyway/models.py b/anyway/models.py index 6ac1a9c19..80f248b9b 100755 --- a/anyway/models.py +++ b/anyway/models.py @@ -4,8 +4,7 @@ import json import logging from collections import namedtuple -from typing import List, Set, Iterable - +from typing import List, Set, Iterable, Optional try: @@ -881,7 +880,9 @@ class NewsFlash(Base): newsflash_location_qualification = Column( Integer(), nullable=False, - server_default=text(f"{NewsflashLocationQualification.NOT_VERIFIED.value}"), # pylint: disable=no-member + server_default=text( + f"{NewsflashLocationQualification.NOT_VERIFIED.value}" + ), # pylint: disable=no-member ) location_qualifying_user = Column(BigInteger(), nullable=True) @@ -897,7 +898,9 @@ def set_critical( ) from anyway.request_params import get_latest_accident_date, LocationInfo - if (self.road1 is None or self.road_segment_id is None) and (self.yishuv_name is None or self.street1_hebrew is None) : + if (self.road1 is None or self.road_segment_id is None) and ( + self.yishuv_name is None or self.street1_hebrew is None + ): return None last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None) resolution = BE_CONST.ResolutionCategories(self.resolution) @@ -911,7 +914,9 @@ def set_critical( location_info["yishuv_name"] = self.yishuv_name location_info["street1_hebrew"] = self.street1_hebrew - critical_values = InjuredCountBySeverityWidget.get_injured_count_by_severity(resolution, location_info, start_time, end_time) + critical_values = InjuredCountBySeverityWidget.get_injured_count_by_severity( + resolution, location_info, start_time, end_time + ) if critical_values == {}: return None critical = None @@ -927,7 +932,6 @@ def set_critical( ) >= 1 self.critical = critical - # generate text describing location or road segment of news flash to be used by # Use case 1 - FE display of curr location in location qualification # Use case 2 - Widgets e.g most severe accidents additional info widget @@ -949,8 +953,10 @@ def get_news_flash_location_text(self): elif resolution == "צומת בינעירוני" and road1 and road_segment_name: res = "כביש " + road1 + " במקטע " + road_segment_name elif resolution == "רחוב" and yishuv_name and street1_hebrew: + def get_street_location_text(yishuv_name, street1_hebrew): return "רחוב " + street1_hebrew + " ב" + yishuv_name + res = get_street_location_text(yishuv_name, street1_hebrew) else: logging.warning( @@ -994,7 +1000,6 @@ def serialize(self): "curr_cbs_location_text": self.get_news_flash_location_text(), } - # Flask-Login integration def is_authenticated(self): return True @@ -1093,11 +1098,19 @@ class City(CityFields, Base): __tablename__ = "cbs_cities" @staticmethod - def get_name_from_symbol(symbol: int) -> str: - res = db.session.query(City.heb_name).filter(City.yishuv_symbol == symbol).first() + def get_name_from_symbol(symbol: int, lang: str = 'he') -> str: + int_sym = int(symbol) + res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int_sym).first() if res is None: - raise ValueError(f"{symbol}: could not find city with that symbol") - return res.heb_name + raise ValueError(f"{int_sym}({symbol}): could not find city with that symbol") + return res.heb_name if lang == 'he' else res.eng_name + + @staticmethod + def get_name_from_symbol_or_none(symbol: int, lang: str = 'he') -> Optional[str]: + try: + return City.get_name_from_symbol(symbol, lang) + except ValueError: + return None @staticmethod def get_symbol_from_name(name: str) -> int: @@ -1117,10 +1130,6 @@ def get_all_cities() -> List[dict]: return res1 -class CityTemp(CityFields, Base): - __tablename__ = "cbs_cities_temp" - - class DeprecatedCity(Base): __tablename__ = "cities" id = Column(Integer(), primary_key=True) @@ -1236,7 +1245,6 @@ def get_streets_by_yishuv(yishuv_symbol: int) -> List[dict]: raise RuntimeError(f"When retrieving streets of {yishuv_symbol}") return res1 - @staticmethod def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]: yishuv_symbol = City.get_symbol_from_name(yishuv_name) @@ -1255,45 +1263,58 @@ class SuburbanJunction(Base): __tablename__ = "suburban_junction" MAX_NAME_LEN = 100 non_urban_intersection = Column(Integer(), primary_key=True, nullable=False) - non_urban_intersection_hebrew = Column(String(length=MAX_NAME_LEN), - nullable=True) + non_urban_intersection_hebrew = Column(String(length=MAX_NAME_LEN), nullable=True) roads = Column(postgresql.ARRAY(Integer(), dimensions=1), nullable=False) @staticmethod def get_hebrew_name_from_id(non_urban_intersection: int) -> str: - res = db.session.query(SuburbanJunction.non_urban_intersection_hebrew).filter( - SuburbanJunction.non_urban_intersection == non_urban_intersection).first() + res = ( + db.session.query(SuburbanJunction.non_urban_intersection_hebrew) + .filter(SuburbanJunction.non_urban_intersection == non_urban_intersection) + .first() + ) if res is None: - raise ValueError(f"{non_urban_intersection}: could not find " - f"SuburbanJunction with that symbol") + raise ValueError( + f"{non_urban_intersection}: could not find " f"SuburbanJunction with that symbol" + ) return res.non_urban_intersection_hebrew @staticmethod def get_id_from_hebrew_name(non_urban_intersection_hebrew: str) -> int: - res = db.session.query(SuburbanJunction.non_urban_intersection).filter( - SuburbanJunction.non_urban_intersection == non_urban_intersection_hebrew).first() + res = ( + db.session.query(SuburbanJunction.non_urban_intersection) + .filter(SuburbanJunction.non_urban_intersection == non_urban_intersection_hebrew) + .first() + ) if res is None: - raise ValueError(f"{non_urban_intersection_hebrew}: could not find " - f"SuburbanJunction with that name") + raise ValueError( + f"{non_urban_intersection_hebrew}: could not find " + f"SuburbanJunction with that name" + ) return res.non_urban_intersection @staticmethod def get_intersection_from_roads(roads: Set[int]) -> dict: if not all([isinstance(x, int) for x in roads]): raise ValueError(f"{roads}: Should be integers") - res = db.session.query(SuburbanJunction).filter( - SuburbanJunction.roads.contains(roads)).first() + res = ( + db.session.query(SuburbanJunction) + .filter(SuburbanJunction.roads.contains(roads)) + .first() + ) if res is None: - raise ValueError(f"{roads}: could not find " - f"SuburbanJunction with these roads") + raise ValueError(f"{roads}: could not find " f"SuburbanJunction with these roads") return res.serialize() @staticmethod def get_all_from_key_value(key: str, val: Iterable) -> dict: if not isinstance(val, Iterable): val = [val] - res = db.session.query(SuburbanJunction).filter( - (getattr(SuburbanJunction, key)).in_(val)).first() + res = ( + db.session.query(SuburbanJunction) + .filter((getattr(SuburbanJunction, key)).in_(val)) + .first() + ) if res is None: raise ValueError(f"{key}:{val}: could not find SuburbanJunction") return res.serialize() @@ -2287,20 +2308,30 @@ def get_segment_id(self): @staticmethod def get_segments_by_segment(road_segment_id: int): - curr_road = (db.session.query(RoadSegments.road) - .filter(RoadSegments.segment_id == road_segment_id) - .all()) + curr_road = ( + db.session.query(RoadSegments.road) + .filter(RoadSegments.segment_id == road_segment_id) + .all() + ) curr_road_processed = [{"road": s.road} for s in curr_road] if curr_road is None or curr_road_processed is None: raise RuntimeError(f"When retrieving segments of {road_segment_id}") road = curr_road_processed[0]["road"] - res = (db.session.query(RoadSegments.segment_id, RoadSegments.from_name, RoadSegments.to_name) - .filter(RoadSegments.road == road) - .all()) - res1 = [{"road": road, "road_segment_id": s.segment_id, "road_segment_name": " - ".join([s.from_name, s.to_name])} for s in res] + res = ( + db.session.query(RoadSegments.segment_id, RoadSegments.from_name, RoadSegments.to_name) + .filter(RoadSegments.road == road) + .all() + ) + res1 = [ + { + "road": road, + "road_segment_id": s.segment_id, + "road_segment_name": " - ".join([s.from_name, s.to_name]), + } + for s in res + ] return res1 - @staticmethod def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]: yishuv_symbol = City.get_symbol_from_name(yishuv_name) @@ -2314,6 +2345,7 @@ def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]: raise RuntimeError(f"When retrieving streets of {yishuv_symbol}") return res1 + class Comment(Base): __tablename__ = "comments" id = Column(BigInteger(), autoincrement=True, primary_key=True, index=True) @@ -2321,7 +2353,7 @@ class Comment(Base): parent = Column(Integer, ForeignKey("comments.id"), nullable=True) created_time = Column(DateTime, default=datetime.datetime.now, index=True, nullable=False) street = Column(Text(), nullable=True, index=True) - city = Column(Text(), nullable=True, index=True) + city = Column(Text(), nullable=True, index=True) road_segment_id = Column(Integer(), nullable=True, index=True) def serialize(self): @@ -2332,12 +2364,10 @@ def serialize(self): "street": self.street, "parent": self.parent, "city": self.city, - "road_segment_id": self.road_segment_id - + "road_segment_id": self.road_segment_id, } - class ReportProblem(Base): __tablename__ = "report_problem" id = Column(BigInteger(), autoincrement=True, primary_key=True, index=True) @@ -2367,7 +2397,7 @@ class InvolvedMarkerView(Base): __table_args__ = ( Index("inv_markers_accident_yishuv_symbol_idx", "accident_yishuv_symbol", unique=False), Index("inv_markers_injury_severity_idx", "injury_severity", unique=False), - Index("inv_markers_involve_vehicle_type_idx", "involve_vehicle_type", unique=False) + Index("inv_markers_involve_vehicle_type_idx", "involve_vehicle_type", unique=False), ) accident_id = Column(BigInteger(), primary_key=True) @@ -3035,14 +3065,17 @@ class TelegramGroupsBase(Base): id = Column(Integer(), primary_key=True) filter = Column(JSON(), nullable=False, server_default="{}") + class TelegramGroups(TelegramGroupsBase): __tablename__ = "telegram_groups" + class TelegramGroupsTest(TelegramGroupsBase): __tablename__ = "telegram_groups_test" + class TelegramForwardedMessages(Base): - __tablename__ = 'telegram_forwarded_messages' + __tablename__ = "telegram_forwarded_messages" message_id = Column(String(), primary_key=True) newsflash_id = Column(BigInteger(), nullable=False) group_sent = Column(String(), nullable=False) diff --git a/anyway/parsers/cbs/executor.py b/anyway/parsers/cbs/executor.py index d5fc2d095..409e94b58 100644 --- a/anyway/parsers/cbs/executor.py +++ b/anyway/parsers/cbs/executor.py @@ -81,6 +81,7 @@ InvolvedMarkerView, VehiclesView, VehicleMarkerView, + City, ) from anyway.parsers.cbs.exceptions import CBSParsingFailed from anyway.utilities import ItmToWGS84, time_delta, ImporterUI, truncate_tables, delete_all_rows_from_table, \ @@ -278,7 +279,7 @@ def get_address(accident, streets): and int(accident.get(field_names.house_number)) != 9999 else None ) - settlement = localization.get_city_name(accident.get(field_names.yishuv_symbol)) + settlement = City.get_name_from_symbol_or_none(accident.get(field_names.yishuv_symbol)) if not house_number and not settlement: return street @@ -522,7 +523,7 @@ def create_marker(provider_code, accident, streets, roads, non_urban_intersectio "km_raw": get_data_value(accident.get(field_names.km)), "km_accurate": km_accurate, "yishuv_symbol": get_data_value(accident.get(field_names.yishuv_symbol)), - "yishuv_name": localization.get_city_name(accident.get(field_names.yishuv_symbol)), + "yishuv_name": City.get_name_from_symbol_or_none(accident.get(field_names.yishuv_symbol)), "geo_area": get_data_value(accident.get(field_names.geo_area)), "day_night": get_data_value(accident.get(field_names.day_night)), "day_in_week": get_data_value(accident.get(field_names.day_in_week)), @@ -602,7 +603,7 @@ def import_involved(provider_code, involved, **kwargs): "involve_yishuv_symbol": get_data_value( involve.get(field_names.involve_yishuv_symbol) ), - "involve_yishuv_name": localization.get_city_name( + "involve_yishuv_name": City.get_name_from_symbol_or_none( involve.get(field_names.involve_yishuv_symbol) ), "injury_severity": get_data_value(involve.get(field_names.injury_severity)), diff --git a/anyway/parsers/cbs/preprocessing_cbs_files.py b/anyway/parsers/cbs/preprocessing_cbs_files.py index 98728adf5..48bd3e979 100644 --- a/anyway/parsers/cbs/preprocessing_cbs_files.py +++ b/anyway/parsers/cbs/preprocessing_cbs_files.py @@ -1,8 +1,4 @@ import os -import logging -import pandas as pd -from anyway.app_and_db import db -from anyway.models import City, CityTemp CBS_FILES_HEBREW = { "sadot": "Fields", @@ -31,74 +27,5 @@ def get_accidents_file_data(directory): return os.path.join(directory, file_path) -# noinspection SpellCheckingInspection -def load_cities_data(file_name: str): - logging.info(f"Loading {file_name} into {City.__tablename__} table.") - column_names = [ - "heb_name", - "yishuv_symbol", - "eng_name", - "district", - "napa", - "natural_zone", - "municipal_stance", - "metropolitan", - "religion", - "population", - "other", - "jews", - "arab", - "founded", - "tzura", - "irgun", - "center", - "altitude", - "planning", - "police", - "year", - "taatik", - ] - col_types = dict.fromkeys(["heb_name", "eng_name", "taatik"], lambda x: str(x) if x else None) - col_types.update( - dict.fromkeys( - [ - "yishuv_symbol", - "district", - "napa", - "natural_zone", - "municipal_stance", - "metropolitan", - "religion", - "population", - "founded", - "tzura", - "irgun", - "center", - "altitude", - "planning", - "police", - "year", - ], - lambda x: int(x) if x and x.isdigit() else 1, - ) - ) - col_types.update(dict.fromkeys(["other", "jews", "arab"], lambda x: float(x) if x else 0.0001)) - cities = pd.read_csv( - file_name, header=0, names=column_names, converters=col_types, encoding="utf-8" - ) - cities_list = cities.to_dict(orient="records") - logging.info(f"Read {len(cities_list)} from {file_name}") - db.session.commit() - db.session.execute("DROP table IF EXISTS cbs_cities_temp") - db.session.execute("CREATE TABLE cbs_cities_temp AS TABLE cbs_cities with NO DATA") - db.session.execute(CityTemp.__table__.insert(), cities_list) - db.session.execute("TRUNCATE table cbs_cities") - db.session.execute("INSERT INTO cbs_cities SELECT * FROM cbs_cities_temp") - db.session.execute("DROP table cbs_cities_temp") - db.session.commit() - num_items = db.session.query(City).count() - logging.info(f"num items in cities: {num_items}.") - - if __name__ == "__main__": pass diff --git a/anyway/parsers/cities.py b/anyway/parsers/cities.py new file mode 100644 index 000000000..da658b49b --- /dev/null +++ b/anyway/parsers/cities.py @@ -0,0 +1,81 @@ +import requests +import json +from typing import Iterable, Dict, Any, List +from anyway.models import City +from anyway.app_and_db import db +import logging + +DATA_GOV_CITIES_RESOURCES_URL = "https://data.gov.il/dataset/citiesandsettelments" +DATA_GOV_CITIES_RESOURCES_ID = "8f714b6f-c35c-4b40-a0e7-547b675eee0e" +RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן" +BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321" +RESOURCE_DOWNLOAD_TEMPLATE = ( + "https://data.gov.il/api/3/action/datastore_search?resource_id={id}&limit=100000" +) +FIELD_NAMES = { + "yishuv_symbol": "city_code", + "heb_name": "city_name_he", + "eng_name": "city_name_en", + "district": "region_code", + "napa": "region_code", + "municipal_stance": "Regional_Council_name", +} +CITY_CODE = "city_code" +CITY_NAME = "city_name_he" +CITY_NAME_EN = "city_name_en" +YISHUV_SYMBOL = "city_code" +MUNICIPAL_STANCE = "PIBA_bureau_code" +NAPA = "Regional_Council_code" +CHUNK_SIZE = 1000 + + +class UpdateCitiesFromDataGov: + def __init__(self): + self.s = requests.Session() + + def get_streets_download_url(self): + url = RESOURCE_DOWNLOAD_TEMPLATE.format(id=DATA_GOV_CITIES_RESOURCES_ID) + return url + + def get_city_data_chunks(self, url: str, chunk_size: int) -> Iterable[List[Dict[str, Any]]]: + r = self.s.get(url) + if not r.ok: + raise Exception(f"Could not get streets url. reason:{r.reason}:{r.status_code}") + r.encoding = "utf-8" + data = json.loads(r.text) + chunk = [] + logging.debug(f"read {len(data['result']['records'])} records from {url}.") + for item in data["result"]["records"]: + city_entry = { + "heb_name": item[CITY_NAME], + "yishuv_symbol": item[YISHUV_SYMBOL], + "eng_name": item[CITY_NAME_EN], + # "napa": item[NAPA], + # "municipal_stance": item[MUNICIPAL_STANCE], + } + chunk.append(city_entry) + if len(chunk) == chunk_size: + yield chunk + chunk = [] + if chunk: + logging.debug(f"last chunk: {len(chunk)}.") + yield chunk + + def import_citis_into_db(self, url: str, chunk_size: int): + num = 0 + db.session.query(City).delete() + for chunk in self.get_city_data_chunks(url=url, chunk_size=chunk_size): + db.session.bulk_insert_mappings(City, chunk) + num += len(chunk) + db.session.commit() + logging.info(f"{num} records written to City table.") + + +def parse(chunk_size=CHUNK_SIZE): + instance = UpdateCitiesFromDataGov() + res = instance.get_streets_download_url() + instance.import_citis_into_db(res, chunk_size) + + +if __name__ == "__main__": + parse() diff --git a/anyway/parsers/streets.py b/anyway/parsers/streets.py index 30d8cc6ae..33f312ccd 100644 --- a/anyway/parsers/streets.py +++ b/anyway/parsers/streets.py @@ -5,25 +5,25 @@ from anyway.app_and_db import db import logging -CBS_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321" +DATA_GOV_STREETS_RESOURCES_URL = "https://data.gov.il/api/3/action/package_show?id=321" RESOURCE_NAME = "רשימת רחובות בישראל - מתעדכן" BASE_GET_DATA_GOV = "https://data.gov.il/dataset/321" RESOURCE_DOWNLOAD_TEMPLATE = ( "https://data.gov.il/api/3/action/datastore_search?resource_id={id}&limit=1000000" ) -STREETS_FIlE_YISHUV_NAME = "שם_ישוב" -STREETS_FIlE_YISHUV_SYMBOL = "סמל_ישוב" -STREETS_FIlE_STREET_NAME = "שם_רחוב" -STREETS_FIlE_STREET_SYMBOL = "סמל_רחוב" +STREETS_FILE_YISHUV_NAME = "שם_ישוב" +STREETS_FILE_YISHUV_SYMBOL = "סמל_ישוב" +STREETS_FILE_STREET_NAME = "שם_רחוב" +STREETS_FILE_STREET_SYMBOL = "סמל_רחוב" CHUNK_SIZE = 1000 -class UpdateStreetsFromCSB: +class UpdateStreetsFromDataGov: def __init__(self): self.s = requests.Session() - def get_cbs_streets_download_url(self): - response = self.s.get(CBS_STREETS_RESOURCES_URL) + def get_streets_download_url(self): + response = self.s.get(DATA_GOV_STREETS_RESOURCES_URL) if not response.ok: raise Exception( f"Could not get streets url. reason:{response.reason}:{response.status_code}" @@ -55,11 +55,11 @@ def get_streets_data_chunks(self, url: str, chunk_size: int) -> Iterable[List[Di chunk = [] logging.debug(f"read {len(data['result']['records'])} records from {url}.") for item in data["result"]["records"]: - street_name = item[STREETS_FIlE_STREET_NAME] + street_name = item[STREETS_FILE_STREET_NAME] street_name_len = len(street_name) street_entry = { - "yishuv_symbol": item[STREETS_FIlE_YISHUV_SYMBOL], - "street": item[STREETS_FIlE_STREET_SYMBOL], + "yishuv_symbol": item[STREETS_FILE_YISHUV_SYMBOL], + "street": item[STREETS_FILE_STREET_SYMBOL], "street_hebrew": street_name[: min(street_name_len, Streets.MAX_NAME_LEN)], } chunk.append(street_entry) @@ -81,8 +81,8 @@ def import_street_file_into_db(self, url: str, chunk_size: int): def parse(chunk_size=CHUNK_SIZE): - instance = UpdateStreetsFromCSB() - res = instance.get_cbs_streets_download_url() + instance = UpdateStreetsFromDataGov() + res = instance.get_streets_download_url() instance.import_street_file_into_db(res, chunk_size) diff --git a/main.py b/main.py index b5ce3d20a..91c868748 100755 --- a/main.py +++ b/main.py @@ -143,7 +143,9 @@ def road_segments(filename): @process.command() -@click.argument("filename", type=str, default="static/data/suburban_junctions/suburban_junctions.xlsx") +@click.argument( + "filename", type=str, default="static/data/suburban_junctions/suburban_junctions.xlsx" +) def suburban_junctions(filename): """Update suburban_junction table from xlsx file""" from anyway.parsers.suburban_junctions import parse @@ -154,12 +156,21 @@ def suburban_junctions(filename): @process.command() @click.argument("chunk-size", type=int, default=1000) def streets(chunk_size): - """Update streets table from CBS site""" + """Update streets table from data.gov site""" from anyway.parsers.streets import parse return parse(chunk_size) +@process.command() +@click.argument("chunk-size", type=int, default=1000) +def cities(chunk_size): + """Update cities table from data.gov site""" + from anyway.parsers.cities import parse + + return parse(chunk_size) + + @process.command() @click.argument("filepath", type=str, default="static/data/schools/schools.csv") @click.option("--batch_size", type=int, default=5000) @@ -178,7 +189,7 @@ def schools(filepath, batch_size): ) @click.option("--batch_size", type=int, default=5000) def schools_with_description( - schools_description_filepath, schools_coordinates_filepath, batch_size + schools_description_filepath, schools_coordinates_filepath, batch_size ): from anyway.parsers.schools_with_description import parse @@ -202,7 +213,7 @@ def schools_with_description( ) @click.option("--batch_size", type=int, default=5000) def schools_with_description_2020( - schools_description_filepath, schools_coordinates_filepath, batch_size + schools_description_filepath, schools_coordinates_filepath, batch_size ): from anyway.parsers.schools_with_description_2020 import parse @@ -224,6 +235,7 @@ def schools_with_description_2020( @click.option("--batch_size", type=int, default=5000) def injured_around_schools_2022(start_date, end_date, distance, batch_size): from anyway.parsers.injured_around_schools_2022 import parse + return parse(start_date=start_date, end_date=end_date, distance=distance, batch_size=batch_size) @@ -238,6 +250,7 @@ def injured_around_schools_2022(start_date, end_date, distance, batch_size): @click.option("--batch_size", type=int, default=5000) def injured_around_schools_2023(start_date, end_date, distance, batch_size): from anyway.parsers.injured_around_schools_2023 import parse + return parse(start_date=start_date, end_date=end_date, distance=distance, batch_size=batch_size) @@ -309,9 +322,7 @@ def infographics_data_cache_for_road_segments(): @process.command() -@click.option( - "--id", type=int, help="newsflash id" -) +@click.option("--id", type=int, help="newsflash id") def infographics_pictures(id): from anyway.infographic_image_generator import generate_infographics_for_newsflash @@ -350,11 +361,12 @@ def update_casualties_costs(filename): return parse(filename) + @process.command() def compare_cbs_anyway(): from anyway.parsers.compare_cbs_and_anyway_road_segments_accidents import parse - return parse() + return parse() @cli.group() @@ -382,19 +394,6 @@ def create_cbs_tables(): return create_tables() -@create_tables.command() -@click.option( - "--file-name", - type=str, - help="csv file to load from. Default is static/data/cities.csv", - default="%s/static/data/cities.csv" % os.path.abspath(os.path.dirname(__file__)), -) -def update_cities_table(file_name): - from anyway.parsers.cbs.preprocessing_cbs_files import load_cities_data - - return load_cities_data(file_name=file_name) - - @cli.group() def update_dictionary_tables(): pass @@ -479,10 +478,10 @@ def accidents_around_schools(start_date, end_date, distance, output_path): @scripts.command() def test_airflow(): - print('my print') - logging.info('info log') - logging.warning('warning log') - logging.debug('debug log') + print("my print") + logging.info("info log") + logging.warning("warning log") + logging.debug("debug log") @scripts.command() @@ -497,9 +496,7 @@ def upload(): pass -@click.option( - "--id", type=int, help="newsflash id" -) +@click.option("--id", type=int, help="newsflash id") @click.option( "--download", is_flag=True, @@ -549,6 +546,7 @@ def generate_images_and_send_notification(id, chat): @click.option("--id", type=int) def trigger_dag(id): from anyway.utilities import trigger_airflow_dag + dag_conf = {"news_flash_id": id} trigger_airflow_dag("generate-and-send-infographics-images", dag_conf) diff --git a/tests/unit_tests/test_localizations.py b/tests/unit_tests/test_localizations.py deleted file mode 100644 index 8635c7855..000000000 --- a/tests/unit_tests/test_localizations.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest - -from anyway.localization import get_city_name - -RAMAT_GAN_CITY_CODE = 8600 - - -class LocalizationsTest(unittest.TestCase): - def test_hebrew(self): - self.assertEqual(get_city_name(RAMAT_GAN_CITY_CODE, "he"), "רמת גן") - - def test_english(self): - self.assertEqual(get_city_name(RAMAT_GAN_CITY_CODE, "en"), "RAMAT GAN") - - def test_not_found(self): - self.assertIsNone(get_city_name("NOT_AN_ACTUAL_CODE", "en"))