rebuild to fix double counting bug

Author: mem48

RScripts/build_passender_od.R

@@ -3,8 +3,8 @@ library(sf)
 library(tidyr)
 
 airports <- read_sf("data/airports_pass.gpkg")
-pass_int_od <- readRDS("data/CAA_int_od_clean.Rds")
-pass_dom_od <- readRDS("data/CAA_dom_od_clean.Rds")
+pass_int_od <- readRDS("data/CAA_int_od_clean_v2.Rds")
+pass_dom_od <- readRDS("data/CAA_dom_od_clean_v2.Rds")
 
 
 head(pass_dom_od)
@@ -37,10 +37,10 @@ pass_od <- pass_od[,c("year","airport1","airport1_country","airport2","airport2_
 # come from two airports that seem to be the same location
 
 pass_od <- pass_od %>%
-  group_by(year,airport1,airport1_country,airport2,airport2_country,airport2) %>%
+  group_by(year,airport1,airport1_country,airport2,airport2_country) %>%
   summarise(total_pax = sum(total_pax))
 
 
 pass_od_wide <- pivot_wider(pass_od, names_from = "year", values_from = "total_pax")
 
-saveRDS(pass_od_wide, "data/passenger_od_wide.Rds")
+saveRDS(pass_od_wide, "data/passenger_od_wide_v2.Rds")

RScripts/clean_passenger_data.R

@@ -6,8 +6,8 @@ tmap_mode("view")
 # Read in
 
 #pass_transit <- readRDS("data/CAA_transit.Rds")
-pass_int_od <- readRDS("data/CAA_int_od.Rds")
-pass_dom_od <- readRDS("data/CAA_dom_od.Rds")
+pass_int_od <- readRDS("data/CAA_int_od_v2.Rds")
+pass_dom_od <- readRDS("data/CAA_dom_od_v2.Rds")
 
 airports_fixed <- readRDS("airports_final.Rds")
 airports2 <- readRDS("airports.Rds")
@@ -458,9 +458,9 @@ pass_dom_od <- pass_dom_od %>%
 pass_dom_od$key <- NULL
 
 
-write_sf(airports_all,"data/airports_pass.gpkg")
-saveRDS(pass_int_od, "data/CAA_int_od_clean.Rds")
-saveRDS(pass_dom_od, "data/CAA_dom_od_clean.Rds")
+write_sf(airports_all,"data/airports_pass_v2.gpkg")
+saveRDS(pass_int_od, "data/CAA_int_od_clean_v2.Rds")
+saveRDS(pass_dom_od, "data/CAA_dom_od_clean_v2.Rds")
 
 qtm(airports_all)
 

RScripts/import_airport_data.R

@@ -157,6 +157,7 @@ for(i in 1:length(zips)){
 
 for(i in seq(length(zips) + 1, length(zips) + 4)){
   dir <- i - length(zips) + 2014
+  message(dir)
   files_csv <- list.files(file.path(path, dir), pattern = ".csv", full.names = TRUE)
   files <- files_csv
 
@@ -189,10 +190,18 @@ for(i in seq(length(zips) + 1, length(zips) + 4)){
   }
   names(int_od) <- c("year","uk_airport","foreign_country","foreign_airport","total_pax", "scheduled_pax","charter_pax")
 
-  dom_od <- files[grep("dom_air_pax_route", files,ignore.case = TRUE)] 
+  dom_od <- files[grep("domestic_air_pax_traffic_route", files,ignore.case = TRUE)] 
   if(length(dom_od) > 1){
     dom_od <- dom_od[1]
   }
+  
+  if(length(dom_od) == 0){
+    dom_od <- files[grep("dom_air_pax_route", files,ignore.case = TRUE)] 
+    if(length(dom_od) > 1){
+      dom_od <- dom_od[1]
+    }
+  }
+  
   dom_od <- readr::read_csv(dom_od)
   if(all(c("this_period","apt1_apt_name","apt2_apt_name","total_pax_tp", "total_pax_shd_tp", "total_pax_cht_tp") %in% names(dom_od))){
     dom_od <- dom_od[,c("this_period","apt1_apt_name","apt2_apt_name","total_pax_tp", "total_pax_shd_tp", "total_pax_cht_tp")]
@@ -222,6 +231,6 @@ transit <- dplyr::bind_rows(res_transit)
 int_od <- dplyr::bind_rows(res_int_od)
 dom_od <- dplyr::bind_rows(res_dom_od[lengths(res_dom_od) >0])
 
-saveRDS(transit, "data/CAA_transit.Rds")
-saveRDS(int_od, "data/CAA_int_od.Rds")
-saveRDS(dom_od, "data/CAA_dom_od.Rds")
+saveRDS(transit, "data/CAA_transit_v2.Rds")
+saveRDS(int_od, "data/CAA_int_od_v2.Rds")
+saveRDS(dom_od, "data/CAA_dom_od_v2.Rds")

RScripts/merge_pass_flight_ods.R

@@ -4,7 +4,7 @@ library(tmap)
 tmap_mode("view")
 
 # Load Passenger OD Data
-pass_od <- readRDS("data/passenger_od_wide.Rds")
+pass_od <- readRDS("data/passenger_od_wide_v2.Rds")
 pass_od <- pass_od[!pass_od$airport2 %in% c("=","Unknown"),]
 pass_od$airport1 <- gsub(" International","",pass_od$airport1)
 pass_od$airport2 <- gsub(" International","",pass_od$airport2)

RScripts/new_method/OD2linestring.R

@@ -3,8 +3,8 @@ library(tmap)
 library(dplyr)
 tmap_mode("view")
 
-od <- readRDS("data/clean/pass_flighs_od.Rds")
-airports <- read_sf("data/clean/airports_clean_second_pass.gpkg")
+od <- readRDS("data/clean/pass_flighs_od_v2.Rds")
+airports <- read_sf("data/clean/airports_clean_second_pass_v2.gpkg")
 
 
 qtm(airports)
@@ -53,13 +53,18 @@ line <- do.call(c, line)
 line <- st_segmentize(line, units::set_units(1, km))
 od_good <- as.data.frame(od_good)
 od_good$geometry <- line
+#stop()
+#od_good$length2 <- round(geodist::geodist(st_coordinates(od_good$geom_from), st_coordinates(od_good$geom_to), paired = TRUE, measure = "geodesic") / 1000, 1)
 od_good$geom_from <- NULL
 od_good$geom_to <- NULL
 
 od_good <- st_as_sf(od_good, crs = 4326)
 
 od_good$length_km <- round(as.numeric(st_length(od_good)) / 1000, 1)
 
+ 
+
+
 od_good$pass_km_2018 <- od_good$`2018` * od_good$length_km
 
 od_top <- od_good[!is.na(od_good$pass_km_2018),]
@@ -101,4 +106,4 @@ ggplot(paris_sum, aes(Year, Passengers, colour = Airport)) +
   scale_x_continuous(breaks = seq(1990,2018,2))
 
 qtm(od_top, lines.col = "pass_km_2018")
-write_sf(od_good, "data/clean/od_flights_pass.gpkg")
+write_sf(od_good, "data/clean/od_flights_pass_v2.gpkg")

RScripts/new_method/combine_flights_passengers.R

@@ -1,7 +1,7 @@
 library(sf)
 library(dplyr)
 
-pass_od <- readRDS("data/clean/passenger_od_wide.Rds")
+pass_od <- readRDS("data/passenger_od_wide_v2.Rds")
 flight_od <- readRDS("data/clean/flights_od_prepped.Rds")
 
 # Load Airports
@@ -447,6 +447,6 @@ flight_od2 <- flight_od %>%
 pass_od2 <- pass_od2[rowSums(pass_od2[,as.character(1990:2018)]) != 0,]
 flight_od2 <- flight_od2[rowSums(flight_od2[,paste0("flt_",1990:2018)]) != 0,]
 
-saveRDS(pass_od2, "data/clean/passenger_od_first_clean.Rds")
-saveRDS(flight_od2, "data/clean/flighs_od_first_clean.Rds")
-write_sf(airports, "data/clean/airports_clean_first_pass.gpkg")
+saveRDS(pass_od2, "data/clean/passenger_od_first_clean_v2.Rds")
+saveRDS(flight_od2, "data/clean/flighs_od_first_clean_v2.Rds")
+write_sf(airports, "data/clean/airports_clean_first_pass_v2.gpkg")

RScripts/new_method/combine_flights_passengers_secondpass.R

@@ -2,19 +2,20 @@
 library(sf)
 library(dplyr)
 
-pass_od <- readRDS("data/clean/passenger_od_first_clean.Rds")
-flight_od <- readRDS("data/clean/flighs_od_first_clean.Rds")
-airports <- read_sf("data/clean/airports_clean_first_pass.gpkg")
+pass_od <- readRDS("data/clean/passenger_od_first_clean_v2.Rds")
+flight_od <- readRDS("data/clean/flighs_od_first_clean_v2.Rds")
+airports <- read_sf("data/clean/airports_clean_first_pass_v2.gpkg")
 airports <- airports[airports$airport != "Vary (Chalons Sur Marne)",]
 
-airports_extra <- data.frame(airport = c("Tabarka","St Maarten","Sydney Canada","Lanseria","Izmir (Cumaovasi)"),
-                             country = c("Tunisia","St Maarten","Canada","Republic of South Africa","Turkey"),
+airports_extra <- data.frame(airport = c("Tabarka","St Maarten","Sydney Canada","Lanseria","Izmir (Cumaovasi)","Islamabad"),
+                             country = c("Tunisia","St Maarten","Canada","Republic of South Africa","Turkey","Pakistan"),
                              geometry = st_sfc(list(
                                st_point(c(8.876944, 36.98)),
                                st_point(c(-63.109444, 18.040833)),
                                st_point(c(-60.048056, 46.161389)),
                                st_point(c(27.926111, -25.938611)),
-                               st_point(c(27.33114, 37.95034))
+                               st_point(c(27.33114, 37.95034)),
+                               st_point(c(72.82565, 33.549083))
                              )))
 names(airports_extra) <- names(airports)
 
@@ -29,7 +30,7 @@ airports$geom[airports$airport == "Seoul Afb"] <- st_point(c(127.113889, 37.4458
 airports$geom[airports$airport == "Seoul (Kimpo)"] <- st_point(c(126.790556, 37.558056))
 airports$geom[airports$airport == "Tollerton Nottingham"] <- st_point(c(-1.080855, 52.91872))
 airports$geom[airports$airport == "Vagar"] <- st_point(c(-7.27546, 62.06333))
-airports$geom[airports$airport == "Benazir Bhutto"] <- st_point(c(72.83501, 33.55693))
+airports$geom[airports$airport == "Benazir Bhutto"] <- st_point(c(73.099167, 33.616389))
 airports$geom[airports$airport == "Hong Kong"] <- st_point(c(113.9185, 22.30805))
 airports$geom[airports$airport == "Camp Springs (Andrews Afb)"] <- st_point(c(-76.88363, 38.79652))
 airports$geom[airports$airport == "Oil Rigs"] <- st_point(c(0.953009, 58.238252))
@@ -127,6 +128,37 @@ tidy_airports("Bursa/Yenisehir","Bursa Yenisehir")
 tidy_airports("Corlu (Afb)","Tekirdag (Corlu)")
 tidy_airports("Cagliari","Cagliari (Elmas)")
 
+tidy_airports("Los Angeles International","Los Angeles")
+tidy_airports("Abu Dhabi International","Abu Dhabi")
+tidy_airports("Miami International","Miami")
+tidy_airports("Philadelphia International","Philadelphia")
+
+tidy_airports("Denver International","Denver")
+tidy_airports("Benazir Bhutto International","Benazir Bhutto")
+tidy_airports("Guangzhou Baiyun International","Guangzhou Baiyun")
+tidy_airports("Halifax Int","Halifax")
+tidy_airports("Ottawa International","Ottawa")
+tidy_airports("Tarbes-Lourdes International","Tarbes-Lourdes")
+tidy_airports("Islamabad International","Islamabad")
+tidy_airports("Jakarta (Soekarno-Hatta Intnl)","Jakarta (Soekarno-Hatta)")
+tidy_airports("Jakarta (Soekarno-Hattanl)","Jakarta (Soekarno-Hatta)")
+tidy_airports("Changsha Huanghua International","Changsha Huanghua")
+tidy_airports("Wuhan Tianhe International","Wuhan Tianhe")
+
+tidy_airports("Male International","Male")
+tidy_airports("Chongqing Jiangbei International","Chongqing Jiangbei")
+tidy_airports("Auckland International","Auckland")
+tidy_airports("Phu Quoc International","Phu Quoc")
+tidy_airports("Sanya Phoenix International","Sanya Phoenix")
+tidy_airports("Kharkov Osnova Intl","Kharkov Osnova")
+tidy_airports("Windsor Locks Bradley Intl","Windsor Locks Bradley")
+tidy_airports("East Midlands International","East Midlands")
+tidy_airports("Nottingham East Midlands Int'l","East Midlands")
+tidy_airports("Manston (Kent Int)","Kent")
+tidy_airports("Bali International","Bali")
+tidy_airports("Minot International","Minot")
+tidy_airports("Erbil International","Erbil")
+
 
 pass_od$airport2_country[pass_od$airport2 == "Pristina"] <- "Kosovo"
 pass_od$airport2_country[pass_od$airport2 == "Bahrain"] <- "Bahrain"
@@ -195,11 +227,11 @@ res <- res[lengths(res) != 0]
 
 
 all_od <- full_join(pass_od2, flight_od2, by= c("airport1","airport1_country","airport2","airport2_country") )
-saveRDS(all_od,"data/clean/pass_flighs_od.Rds")
-saveRDS(pass_od2, "data/clean/passenger_od_second_clean.Rds")
-saveRDS(flight_od, "data/clean/flighs_od_second_clean.Rds")
+saveRDS(all_od,"data/clean/pass_flighs_od_v2.Rds")
+saveRDS(pass_od2, "data/clean/passenger_od_second_clean_v2.Rds")
+saveRDS(flight_od, "data/clean/flighs_od_second_clean_v2.Rds")
 airports <- airports[!duplicated(airports),]
-write_sf(airports, "data/clean/airports_clean_second_pass.gpkg")
+write_sf(airports, "data/clean/airports_clean_second_pass_v2.gpkg")
 
 
 # Check Airport Locations