Merge pull request #2680 from ziv17/2665-cities-from-data.gov

2665 cities from data.gov

.github/workflows/Tests.yml

@@ -65,6 +65,8 @@ jobs:
     - name: DB Initialization
       run: |
         docker exec anyway alembic upgrade head
+        docker exec anyway ./main.py process cities
+        docker exec anyway ./main.py process streets
         docker exec anyway ./main.py process registered-vehicles
         docker exec anyway ./main.py process cbs --source local_dir_for_tests_only
         docker exec anyway ./main.py process road-segments

anyway/localization.py

@@ -3,7 +3,6 @@
 import pandas as pd
 
 from anyway import field_names
-from typing import Optional
 
 import logging
 
@@ -214,11 +213,3 @@ def get_field(field, value=None):
 
 def get_supported_tables():
     return _tables.keys()
-
-
-def get_city_name(symbol_id, lang: str = "he") -> Optional[str]:
-    column_to_fetch = field_names.name if lang == "he" else "ENGLISH_NAME"
-    try:
-        return _cities.loc[symbol_id, column_to_fetch]
-    except Exception:
-        return None

anyway/models.py

@@ -4,8 +4,7 @@
 import json
 import logging
 from collections import namedtuple
-from typing import List, Set, Iterable
-
+from typing import List, Set, Iterable, Optional
 
 
 try:
@@ -881,7 +880,9 @@ class NewsFlash(Base):
     newsflash_location_qualification = Column(
         Integer(),
         nullable=False,
-        server_default=text(f"{NewsflashLocationQualification.NOT_VERIFIED.value}"),  # pylint: disable=no-member
+        server_default=text(
+            f"{NewsflashLocationQualification.NOT_VERIFIED.value}"
+        ),  # pylint: disable=no-member
     )
     location_qualifying_user = Column(BigInteger(), nullable=True)
 
@@ -897,7 +898,9 @@ def set_critical(
         )
         from anyway.request_params import get_latest_accident_date, LocationInfo
 
-        if (self.road1 is None or self.road_segment_id is None) and (self.yishuv_name is None or self.street1_hebrew is None) :
+        if (self.road1 is None or self.road_segment_id is None) and (
+            self.yishuv_name is None or self.street1_hebrew is None
+        ):
             return None
         last_accident_date = get_latest_accident_date(table_obj=AccidentMarkerView, filters=None)
         resolution = BE_CONST.ResolutionCategories(self.resolution)
@@ -911,7 +914,9 @@ def set_critical(
             location_info["yishuv_name"] = self.yishuv_name
             location_info["street1_hebrew"] = self.street1_hebrew
 
-        critical_values = InjuredCountBySeverityWidget.get_injured_count_by_severity(resolution, location_info, start_time, end_time)
+        critical_values = InjuredCountBySeverityWidget.get_injured_count_by_severity(
+            resolution, location_info, start_time, end_time
+        )
         if critical_values == {}:
             return None
         critical = None
@@ -927,7 +932,6 @@ def set_critical(
             ) >= 1
         self.critical = critical
 
-
     # generate text describing location or road segment of news flash to be used by
     # Use case 1 - FE display of curr location in location qualification
     # Use case 2 - Widgets e.g most severe accidents additional info widget
@@ -949,8 +953,10 @@ def get_news_flash_location_text(self):
         elif resolution == "צומת בינעירוני" and road1 and road_segment_name:
             res = "כביש " + road1 + " במקטע " + road_segment_name
         elif resolution == "רחוב" and yishuv_name and street1_hebrew:
+
             def get_street_location_text(yishuv_name, street1_hebrew):
                 return "רחוב " + street1_hebrew + " ב" + yishuv_name
+
             res = get_street_location_text(yishuv_name, street1_hebrew)
         else:
             logging.warning(
@@ -994,7 +1000,6 @@ def serialize(self):
             "curr_cbs_location_text": self.get_news_flash_location_text(),
         }
 
-
     # Flask-Login integration
     def is_authenticated(self):
         return True
@@ -1093,11 +1098,19 @@ class City(CityFields, Base):
     __tablename__ = "cbs_cities"
 
     @staticmethod
-    def get_name_from_symbol(symbol: int) -> str:
-        res = db.session.query(City.heb_name).filter(City.yishuv_symbol == symbol).first()
+    def get_name_from_symbol(symbol: int, lang: str = 'he') -> str:
+        int_sym = int(symbol)
+        res: City = db.session.query(City.heb_name, City.eng_name).filter(City.yishuv_symbol == int_sym).first()
         if res is None:
-            raise ValueError(f"{symbol}: could not find city with that symbol")
-        return res.heb_name
+            raise ValueError(f"{int_sym}({symbol}): could not find city with that symbol")
+        return res.heb_name if lang == 'he' else res.eng_name
+
+    @staticmethod
+    def get_name_from_symbol_or_none(symbol: int, lang: str = 'he') -> Optional[str]:
+        try:
+            return City.get_name_from_symbol(symbol, lang)
+        except ValueError:
+            return None
 
     @staticmethod
     def get_symbol_from_name(name: str) -> int:
@@ -1117,10 +1130,6 @@ def get_all_cities() -> List[dict]:
         return res1
 
 
-class CityTemp(CityFields, Base):
-    __tablename__ = "cbs_cities_temp"
-
-
 class DeprecatedCity(Base):
     __tablename__ = "cities"
     id = Column(Integer(), primary_key=True)
@@ -1236,7 +1245,6 @@ def get_streets_by_yishuv(yishuv_symbol: int) -> List[dict]:
             raise RuntimeError(f"When retrieving streets of {yishuv_symbol}")
         return res1
 
-
     @staticmethod
     def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]:
         yishuv_symbol = City.get_symbol_from_name(yishuv_name)
@@ -1255,45 +1263,58 @@ class SuburbanJunction(Base):
     __tablename__ = "suburban_junction"
     MAX_NAME_LEN = 100
     non_urban_intersection = Column(Integer(), primary_key=True, nullable=False)
-    non_urban_intersection_hebrew = Column(String(length=MAX_NAME_LEN),
-                                           nullable=True)
+    non_urban_intersection_hebrew = Column(String(length=MAX_NAME_LEN), nullable=True)
     roads = Column(postgresql.ARRAY(Integer(), dimensions=1), nullable=False)
 
     @staticmethod
     def get_hebrew_name_from_id(non_urban_intersection: int) -> str:
-        res = db.session.query(SuburbanJunction.non_urban_intersection_hebrew).filter(
-            SuburbanJunction.non_urban_intersection == non_urban_intersection).first()
+        res = (
+            db.session.query(SuburbanJunction.non_urban_intersection_hebrew)
+            .filter(SuburbanJunction.non_urban_intersection == non_urban_intersection)
+            .first()
+        )
         if res is None:
-            raise ValueError(f"{non_urban_intersection}: could not find "
-                             f"SuburbanJunction with that symbol")
+            raise ValueError(
+                f"{non_urban_intersection}: could not find " f"SuburbanJunction with that symbol"
+            )
         return res.non_urban_intersection_hebrew
 
     @staticmethod
     def get_id_from_hebrew_name(non_urban_intersection_hebrew: str) -> int:
-        res = db.session.query(SuburbanJunction.non_urban_intersection).filter(
-            SuburbanJunction.non_urban_intersection == non_urban_intersection_hebrew).first()
+        res = (
+            db.session.query(SuburbanJunction.non_urban_intersection)
+            .filter(SuburbanJunction.non_urban_intersection == non_urban_intersection_hebrew)
+            .first()
+        )
         if res is None:
-            raise ValueError(f"{non_urban_intersection_hebrew}: could not find "
-                             f"SuburbanJunction with that name")
+            raise ValueError(
+                f"{non_urban_intersection_hebrew}: could not find "
+                f"SuburbanJunction with that name"
+            )
         return res.non_urban_intersection
 
     @staticmethod
     def get_intersection_from_roads(roads: Set[int]) -> dict:
         if not all([isinstance(x, int) for x in roads]):
             raise ValueError(f"{roads}: Should be integers")
-        res = db.session.query(SuburbanJunction).filter(
-            SuburbanJunction.roads.contains(roads)).first()
+        res = (
+            db.session.query(SuburbanJunction)
+            .filter(SuburbanJunction.roads.contains(roads))
+            .first()
+        )
         if res is None:
-            raise ValueError(f"{roads}: could not find "
-                             f"SuburbanJunction with these roads")
+            raise ValueError(f"{roads}: could not find " f"SuburbanJunction with these roads")
         return res.serialize()
 
     @staticmethod
     def get_all_from_key_value(key: str, val: Iterable) -> dict:
         if not isinstance(val, Iterable):
             val = [val]
-        res = db.session.query(SuburbanJunction).filter(
-            (getattr(SuburbanJunction, key)).in_(val)).first()
+        res = (
+            db.session.query(SuburbanJunction)
+            .filter((getattr(SuburbanJunction, key)).in_(val))
+            .first()
+        )
         if res is None:
             raise ValueError(f"{key}:{val}: could not find SuburbanJunction")
         return res.serialize()
@@ -2287,20 +2308,30 @@ def get_segment_id(self):
 
     @staticmethod
     def get_segments_by_segment(road_segment_id: int):
-        curr_road = (db.session.query(RoadSegments.road)
-                    .filter(RoadSegments.segment_id == road_segment_id)
-                    .all())
+        curr_road = (
+            db.session.query(RoadSegments.road)
+            .filter(RoadSegments.segment_id == road_segment_id)
+            .all()
+        )
         curr_road_processed = [{"road": s.road} for s in curr_road]
         if curr_road is None or curr_road_processed is None:
             raise RuntimeError(f"When retrieving segments of {road_segment_id}")
         road = curr_road_processed[0]["road"]
-        res = (db.session.query(RoadSegments.segment_id, RoadSegments.from_name, RoadSegments.to_name)
-               .filter(RoadSegments.road == road)
-               .all())
-        res1 = [{"road": road, "road_segment_id": s.segment_id, "road_segment_name": " - ".join([s.from_name, s.to_name])} for s in res]
+        res = (
+            db.session.query(RoadSegments.segment_id, RoadSegments.from_name, RoadSegments.to_name)
+            .filter(RoadSegments.road == road)
+            .all()
+        )
+        res1 = [
+            {
+                "road": road,
+                "road_segment_id": s.segment_id,
+                "road_segment_name": " - ".join([s.from_name, s.to_name]),
+            }
+            for s in res
+        ]
         return res1
 
-
     @staticmethod
     def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]:
         yishuv_symbol = City.get_symbol_from_name(yishuv_name)
@@ -2314,14 +2345,15 @@ def get_streets_by_yishuv_name(yishuv_name: str) -> List[dict]:
             raise RuntimeError(f"When retrieving streets of {yishuv_symbol}")
         return res1
 
+
 class Comment(Base):
     __tablename__ = "comments"
     id = Column(BigInteger(), autoincrement=True, primary_key=True, index=True)
     author = Column(Integer(), ForeignKey("users.id"), nullable=False)
     parent = Column(Integer, ForeignKey("comments.id"), nullable=True)
     created_time = Column(DateTime, default=datetime.datetime.now, index=True, nullable=False)
     street = Column(Text(), nullable=True, index=True)
-    city = Column(Text(), nullable=True,  index=True)
+    city = Column(Text(), nullable=True, index=True)
     road_segment_id = Column(Integer(), nullable=True, index=True)
 
     def serialize(self):
@@ -2332,12 +2364,10 @@ def serialize(self):
             "street": self.street,
             "parent": self.parent,
             "city": self.city,
-            "road_segment_id": self.road_segment_id
-
+            "road_segment_id": self.road_segment_id,
         }
 
 
-
 class ReportProblem(Base):
     __tablename__ = "report_problem"
     id = Column(BigInteger(), autoincrement=True, primary_key=True, index=True)
@@ -2367,7 +2397,7 @@ class InvolvedMarkerView(Base):
     __table_args__ = (
         Index("inv_markers_accident_yishuv_symbol_idx", "accident_yishuv_symbol", unique=False),
         Index("inv_markers_injury_severity_idx", "injury_severity", unique=False),
-        Index("inv_markers_involve_vehicle_type_idx", "involve_vehicle_type", unique=False)
+        Index("inv_markers_involve_vehicle_type_idx", "involve_vehicle_type", unique=False),
     )
 
     accident_id = Column(BigInteger(), primary_key=True)
@@ -3035,14 +3065,17 @@ class TelegramGroupsBase(Base):
     id = Column(Integer(), primary_key=True)
     filter = Column(JSON(), nullable=False, server_default="{}")
 
+
 class TelegramGroups(TelegramGroupsBase):
     __tablename__ = "telegram_groups"
 
+
 class TelegramGroupsTest(TelegramGroupsBase):
     __tablename__ = "telegram_groups_test"
 
+
 class TelegramForwardedMessages(Base):
-    __tablename__ = 'telegram_forwarded_messages'
+    __tablename__ = "telegram_forwarded_messages"
     message_id = Column(String(), primary_key=True)
     newsflash_id = Column(BigInteger(), nullable=False)
     group_sent = Column(String(), nullable=False)

anyway/parsers/cbs/executor.py

@@ -81,6 +81,7 @@
     InvolvedMarkerView,
     VehiclesView,
     VehicleMarkerView,
+    City,
 )
 from anyway.parsers.cbs.exceptions import CBSParsingFailed
 from anyway.utilities import ItmToWGS84, time_delta, ImporterUI, truncate_tables, delete_all_rows_from_table, \
@@ -278,7 +279,7 @@ def get_address(accident, streets):
            and int(accident.get(field_names.house_number)) != 9999
         else None
     )
-    settlement = localization.get_city_name(accident.get(field_names.yishuv_symbol))
+    settlement = City.get_name_from_symbol_or_none(accident.get(field_names.yishuv_symbol))
 
     if not house_number and not settlement:
         return street
@@ -522,7 +523,7 @@ def create_marker(provider_code, accident, streets, roads, non_urban_intersectio
         "km_raw": get_data_value(accident.get(field_names.km)),
         "km_accurate": km_accurate,
         "yishuv_symbol": get_data_value(accident.get(field_names.yishuv_symbol)),
-        "yishuv_name": localization.get_city_name(accident.get(field_names.yishuv_symbol)),
+        "yishuv_name": City.get_name_from_symbol_or_none(accident.get(field_names.yishuv_symbol)),
         "geo_area": get_data_value(accident.get(field_names.geo_area)),
         "day_night": get_data_value(accident.get(field_names.day_night)),
         "day_in_week": get_data_value(accident.get(field_names.day_in_week)),
@@ -602,7 +603,7 @@ def import_involved(provider_code, involved, **kwargs):
                 "involve_yishuv_symbol": get_data_value(
                     involve.get(field_names.involve_yishuv_symbol)
                 ),
-                "involve_yishuv_name": localization.get_city_name(
+                "involve_yishuv_name": City.get_name_from_symbol_or_none(
                     involve.get(field_names.involve_yishuv_symbol)
                 ),
                 "injury_severity": get_data_value(involve.get(field_names.injury_severity)),