From 5c91e9e50d0884a1c77ba610e37224ec6a18598e Mon Sep 17 00:00:00 2001 From: David Kane Date: Mon, 19 Jun 2023 11:00:25 +0100 Subject: [PATCH] update urls --- findthatpostcode/commands/codes.py | 12 +++--------- findthatpostcode/commands/placenames.py | 4 +--- findthatpostcode/commands/postcodes.py | 5 ++--- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/findthatpostcode/commands/codes.py b/findthatpostcode/commands/codes.py index 8323de5..1aa3de6 100644 --- a/findthatpostcode/commands/codes.py +++ b/findthatpostcode/commands/codes.py @@ -17,10 +17,10 @@ from flask.cli import with_appcontext from .. import db -from ..metadata import AREA_TYPES, ENTITIES +from ..metadata import ENTITIES -RGC_URL = "https://www.arcgis.com/sharing/rest/content/items/ab6418e607fc416984539ef2c77d8495/data" -CHD_URL = "https://www.arcgis.com/sharing/rest/content/items/908b216ed0db4333bd7ca056c1b5a364/data" +RGC_URL = "https://www.arcgis.com/sharing/rest/content/items/34f4b9d554324bc494dc406dca58001a/data" +CHD_URL = "https://www.arcgis.com/sharing/rest/content/items/393a031178684c69973d0e416a862890/data" MSOA_2011_URL = ( "https://houseofcommonslibrary.github.io/msoanames/MSOA-Names-Latest.csv" ) @@ -61,7 +61,6 @@ def process_float(value): @click.option("--es-index", default=ENTITY_INDEX) @with_appcontext def import_rgc(url=RGC_URL, es_index=ENTITY_INDEX): - if current_app.config["DEBUG"]: requests_cache.install_cache() @@ -76,7 +75,6 @@ def import_rgc(url=RGC_URL, es_index=ENTITY_INDEX): reader = csv.DictReader(io.TextIOWrapper(infile, "utf-8-sig")) entities = [] for entity in reader: - # tidy up a couple of records entity["Related entity codes"] = ( entity["Related entity codes"].replace("n/a", "").split(", ") @@ -139,7 +137,6 @@ def import_rgc(url=RGC_URL, es_index=ENTITY_INDEX): @click.option("--encoding", default=DEFAULT_ENCODING) @with_appcontext def import_chd(url=CHD_URL, es_index=AREA_INDEX, encoding=DEFAULT_ENCODING): - if current_app.config["DEBUG"]: requests_cache.install_cache() @@ -260,7 +257,6 @@ def import_chd(url=CHD_URL, es_index=AREA_INDEX, encoding=DEFAULT_ENCODING): @click.option("--es-index", default=AREA_INDEX) @with_appcontext def import_msoa_names(url=MSOA_2011_URL, es_index=AREA_INDEX): - if current_app.config["DEBUG"]: requests_cache.install_cache() @@ -269,10 +265,8 @@ def import_msoa_names(url=MSOA_2011_URL, es_index=AREA_INDEX): r = requests.get(url, stream=True) reader = csv.DictReader(codecs.iterdecode(r.iter_lines(), "utf-8-sig")) - name_fields = {} area_updates = [] for k, area in tqdm.tqdm(enumerate(reader)): - if "msoa21cd" in area.keys(): areacode = area.get("msoa21cd") name = area.get("msoa21hclnm") diff --git a/findthatpostcode/commands/placenames.py b/findthatpostcode/commands/placenames.py index 2c778b8..35bfe05 100644 --- a/findthatpostcode/commands/placenames.py +++ b/findthatpostcode/commands/placenames.py @@ -16,7 +16,7 @@ PLACENAMES_INDEX = "geo_placename" -PLACENAMES_URL = "https://www.arcgis.com/sharing/rest/content/items/8f8b561f256b40c3a6df71e400bb54f0/data" +PLACENAMES_URL = "https://www.arcgis.com/sharing/rest/content/items/6cb9092a37da4b5ea1b5f8b054c343aa/data" PLACE_TYPES = { "BUA": ["Built-up Area", "England and Wales"], @@ -71,7 +71,6 @@ @click.option("--url", default=PLACENAMES_URL) @with_appcontext def import_placenames(url=PLACENAMES_URL, es_index=PLACENAMES_INDEX): - if current_app.config["DEBUG"]: requests_cache.install_cache() @@ -93,7 +92,6 @@ def import_placenames(url=PLACENAMES_URL, es_index=PLACENAMES_INDEX): place_code = None place_name = None for i in reader: - # get the names of the name and code fields if not place_code or not place_name: for key in i.keys(): diff --git a/findthatpostcode/commands/postcodes.py b/findthatpostcode/commands/postcodes.py index 2230d61..efe21b7 100644 --- a/findthatpostcode/commands/postcodes.py +++ b/findthatpostcode/commands/postcodes.py @@ -19,8 +19,8 @@ PC_INDEX = "geo_postcode" NSPL_URL = { - 2011: "https://www.arcgis.com/sharing/rest/content/items/3685e7b7261b4d6a9fc55d8d1d5054b2/data", - 2021: "https://www.arcgis.com/sharing/rest/content/items/5922269bd3254db7835511f33181ebd3/data", + 2011: "https://www.arcgis.com/sharing/rest/content/items/782899bd01934a8099ae8516cc021f68/data", + 2021: "https://www.arcgis.com/sharing/rest/content/items/b86748732a054592bcf0218e86a43870/data", } DEFAULT_YEAR = 2021 @@ -31,7 +31,6 @@ @click.option("--year", default=DEFAULT_YEAR, type=int) @with_appcontext def import_nspl(url=None, es_index=PC_INDEX, year=DEFAULT_YEAR): - if not url: url = NSPL_URL[year]