From 1672ebc0283884f5f8f02a0c4171647f5e5d874c Mon Sep 17 00:00:00 2001 From: Feleir Date: Sat, 29 Jul 2023 16:58:22 +0200 Subject: [PATCH 1/4] Update sign processing to update traffic signs code #17 Removes Z from the code and adds zone to the parameters Updated join to do a left join --- code/sign_processing.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/code/sign_processing.py b/code/sign_processing.py index cca9d5e..0da595f 100644 --- a/code/sign_processing.py +++ b/code/sign_processing.py @@ -30,20 +30,24 @@ def extract_new_signs(feature_file, process_date): return filtered_df # Convert coordinates filtered_df[['longitude','latitude']] = filtered_df.apply(convertCoords,axis=1) - # Bordcode processing, remove Z from it and add (zone) description. - filtered_df['bordcode'] = filtered_df.apply(lambda row: (f"{row['bordcode'][1:]} (zone)" if row['bordcode'].startswith('Z') else row['bordcode']).replace("/", ""), axis=1) + # Update parameters to add zone if the sign code starts with Z. + filtered_df['parameters'] = filtered_df.apply(lambda row: f"zone,{row['parameters']}" if row['bordcode'].startswith('Z') else row['parameters'], axis=1) + # Removed Z if the bordcode starts with a Z, remove all /s and store it in traffic_sign_code column. + filtered_df['traffic_sign_code'] = filtered_df.apply(lambda row: (f"{row['bordcode'][1:]}" if row['bordcode'].startswith('Z') else row['bordcode']).replace("/", ""), axis=1) # Replace strings from FID filtered_df['id'] = filtered_df['FID'].str.replace('Verkeersborden.Vlaanderen_Borden.','') filtered_df.drop(columns=['FID']) sign_metadata = pd.read_csv(traffic_signs_info, sep=";", encoding = "ISO-8859-1") - # Join both datasets by the bordcode - joined_df = filtered_df.join(sign_metadata.set_index("bordcode"), on='bordcode') + # Rename bordcode to traffic_sign_code. + sign_metadata = sign_metadata.rename(columns={"bordcode": "traffic_sign_code"}) + # Join both datasets by the traffic_sign_code doing a left join. + joined_df = filtered_df.merge(sign_metadata, on='traffic_sign_code', how='left') # Remove NaN parameters and name joined_df[['parameters', 'name']] = joined_df[['parameters','name']].fillna('') # Group the features by pole identifier. grouped_df = joined_df.groupby('id_aanzicht', as_index=False).agg({ 'opinion': 'max', - 'bordcode': ' | '.join, + 'traffic_sign_code': ' | '.join, 'latitude': 'max', 'longitude': 'max', 'parameters': lambda x : '|'.join(y for y in x if y != ''), @@ -54,7 +58,6 @@ def extract_new_signs(feature_file, process_date): logger.debug("Found %d signs after grouping by pole identifier.", len(grouped_df)) # Rename columns as per the geojson requirements return grouped_df.rename(columns = { - "bordcode": "traffic_sign_code", "parameters": "extra_text", "datum_plaatsing": "date_installed", "name": "traffic_sign_description" From c3f1c6325f490867ccf908b265b6f802f51c4921 Mon Sep 17 00:00:00 2001 From: Feleir Date: Sat, 5 Aug 2023 09:17:52 +0200 Subject: [PATCH 2/4] Temporary commit to test github actions --- .github/workflows/on_demand_trigger.yml | 11 +++++++++++ code/process_new_signs.py | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/on_demand_trigger.yml b/.github/workflows/on_demand_trigger.yml index cd1b264..c8868bb 100644 --- a/.github/workflows/on_demand_trigger.yml +++ b/.github/workflows/on_demand_trigger.yml @@ -40,3 +40,14 @@ jobs: run: | python3 code/process_new_signs.py shell: sh + - name: Upload WFS features + uses: actions/upload-artifact@v3 + with: + name: WFS_Input + path: python_output/feature_output.csv + - name: Upload geojson results + uses: actions/upload-artifact@v3 + with: + name: GeoJSON_Output + path: python_output/geojson_output.json + diff --git a/code/process_new_signs.py b/code/process_new_signs.py index 0521d31..78cd88d 100644 --- a/code/process_new_signs.py +++ b/code/process_new_signs.py @@ -11,8 +11,8 @@ feature_file = "./python_output/feature_output.csv" geojson_file = "./python_output/geojson_output.json" process_date = get_process_date() -maproulette_api_key = environ.get("MAPROULETTE_API_KEY") -challenge_id = environ['CHALLENGE_ID'] +#maproulette_api_key = environ.get("MAPROULETTE_API_KEY") +#challenge_id = environ['CHALLENGE_ID'] if __name__== "__main__": logging.info("Processing new features after %s, updates will be published to challenge %s [API key: %s].", process_date, challenge_id, maproulette_api_key) @@ -22,4 +22,4 @@ logging.info("No new features found.") else: save_geojson(signs_dataframe, geojson_file) - upload_to_maproulette(maproulette_api_key, challenge_id, geojson_file) + #upload_to_maproulette(maproulette_api_key, challenge_id, geojson_file) From cb0a1d187ad9d66f303dc0195c8afc24a79dd81b Mon Sep 17 00:00:00 2001 From: Feleir Date: Sat, 5 Aug 2023 09:26:21 +0200 Subject: [PATCH 3/4] Temporary update --- code/process_new_signs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/process_new_signs.py b/code/process_new_signs.py index 78cd88d..6a27e96 100644 --- a/code/process_new_signs.py +++ b/code/process_new_signs.py @@ -15,7 +15,7 @@ #challenge_id = environ['CHALLENGE_ID'] if __name__== "__main__": - logging.info("Processing new features after %s, updates will be published to challenge %s [API key: %s].", process_date, challenge_id, maproulette_api_key) + logging.info("Processing new features after %s, updates will be published.", process_date) fetch_all_features_by_type(wfs_url, feature_file, feature_type) signs_dataframe = extract_new_signs(feature_file, process_date) if signs_dataframe.empty: From f4df5fb25a64df16ab10865144b39ac95922cc3c Mon Sep 17 00:00:00 2001 From: Feleir Date: Sat, 5 Aug 2023 09:57:25 +0200 Subject: [PATCH 4/4] Revert testing changes --- code/process_new_signs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/process_new_signs.py b/code/process_new_signs.py index 6a27e96..0521d31 100644 --- a/code/process_new_signs.py +++ b/code/process_new_signs.py @@ -11,15 +11,15 @@ feature_file = "./python_output/feature_output.csv" geojson_file = "./python_output/geojson_output.json" process_date = get_process_date() -#maproulette_api_key = environ.get("MAPROULETTE_API_KEY") -#challenge_id = environ['CHALLENGE_ID'] +maproulette_api_key = environ.get("MAPROULETTE_API_KEY") +challenge_id = environ['CHALLENGE_ID'] if __name__== "__main__": - logging.info("Processing new features after %s, updates will be published.", process_date) + logging.info("Processing new features after %s, updates will be published to challenge %s [API key: %s].", process_date, challenge_id, maproulette_api_key) fetch_all_features_by_type(wfs_url, feature_file, feature_type) signs_dataframe = extract_new_signs(feature_file, process_date) if signs_dataframe.empty: logging.info("No new features found.") else: save_geojson(signs_dataframe, geojson_file) - #upload_to_maproulette(maproulette_api_key, challenge_id, geojson_file) + upload_to_maproulette(maproulette_api_key, challenge_id, geojson_file)