Moved to Standard, added health check

app/app/src/main/java/io/oisin/fyp/MapsActivity.java

@@ -156,13 +156,34 @@ protected void onCreate(Bundle savedInstanceState) {
             Toast.makeText(getApplicationContext(), "Thank you for your feedback!", Toast.LENGTH_LONG).show();
         }
 
+        pingBackEnd();
         createNotificationChannel();
         setUpMapUI();
         setUpAutocomplete();
         setUpBottomSheet();
         setUpStationTypeButtons();
     }
 
+    // Used to wake up the server in case it's asleep, due to moving from a Flex AppEngine instance
+    // to a Standard AppEngine instance
+    public void pingBackEnd() {
+        // Request a string response from the provided URL.
+        StringRequest stringRequest = new StringRequest(Request.Method.GET, "https://dbikes-planner.appspot.com/health-check",
+                new Response.Listener<String>() {
+                    @Override
+                    public void onResponse(String response) {
+                        Log.i("MapsActivity", "onResponse: Ping to back-end successful.");
+                    }
+                }, new Response.ErrorListener() {
+            @Override
+            public void onErrorResponse(VolleyError error) {
+                Log.w("MapsActivity", "onResponse: Ping to back-end failed.");
+            }
+        });
+
+        requestQueue.add(stringRequest);
+    }
+
     @Override
     public void onBackPressed() {
         View bottomSheet = findViewById(R.id.route_bottom_sheet);
@@ -1298,7 +1319,7 @@ private int getMarkerTotalSpaces(Marker marker) {
         String snippet = marker.getSnippet().split(splitter)[0];
         return Integer.parseInt(snippet.split(" out of ")[1]);
     }
-    
+
     private class StationRenderer extends DefaultClusterRenderer<StationClusterItem> {
 
         StationRenderer() {

webapp/app.yaml

@@ -1,13 +1,3 @@
-runtime: python
-env: flex
-entrypoint: gunicorn -b :$PORT main:app
-
-runtime_config:
-  python_version: 3
-
-manual_scaling:
-  instances: 1
-resources:
-  cpu: 1
-  memory_gb: 0.5
-  disk_size_gb: 10
+runtime: python37
+env: standard
+entrypoint: gunicorn -b :$PORT main:app

webapp/cloud_function.py

@@ -0,0 +1,125 @@
+import json
+
+import pandas as pd
+import requests
+import datalab.storage as gcs
+from datetime import datetime, time
+
+
+def update_bike_data(request):
+    bike_data = pd.read_json(
+        "https://api.jcdecaux.com/vls/v1/stations?contract=dublin&apiKey=6e5c2a98e60a3336ecaede8f8c8688da25144692")
+
+    weather = get_weather()
+
+    print("Updating bike data...")
+    for _index, row in bike_data.iterrows():
+        update_record(row, weather)
+    print("Bike data refresh complete!")
+
+    return "Success!"
+
+
+def get_weather():
+    print("Updating weather data...")
+    js = requests.get(
+        "https://api.openweathermap.org/data/2.5/weather?lat=53.277717&lon=-6.218428&APPID"
+        "=d8d0b9ed5f181cfbdf3330b0037aff7d&units=metric").text
+
+    request_weather = json.loads(js)
+
+    parsed_weather = {'rain': 0.0}
+
+    if "rain" in request_weather and "1h" in request_weather["rain"]:
+        parsed_weather['rain'] = request_weather["rain"]["1h"]
+
+    parsed_weather['temperature'] = request_weather['main']['temp']
+    parsed_weather['humidity'] = request_weather['main']['humidity']
+    parsed_weather['wind_speed'] = request_weather['wind']['speed']
+
+    if "visibility" in request_weather:
+        parsed_weather['visibility'] = request_weather['visibility']
+    else:
+        parsed_weather['visibility'] = 0
+
+    print("Weather data update complete!")
+
+    return parsed_weather
+
+
+def datetime_to_seconds(t):
+    return (t.hour * 60 + t.minute) * 60 + t.second
+
+
+def seconds_in_previous_days(t):
+    return t.timetuple().tm_yday * 24 * 60 * 60
+
+
+def nearest_time(items, pivot):
+    return min(items, key=lambda x: abs(x - pivot))
+
+
+def is_time_between(begin_time, end_time, check_time=None):
+    # If check time is not given, default to current UTC time
+    check_time = check_time or datetime.utcnow().time()
+    if begin_time < end_time:
+        return begin_time <= check_time <= end_time
+    else:  # crosses midnight
+        return check_time >= begin_time or check_time <= end_time
+
+
+def category(quantity):
+    if quantity == 0:
+        return 'empty'
+    elif quantity < 2:
+        return 'very low'
+    elif quantity < 5:
+        return 'low'
+    elif quantity < 10:
+        return 'moderate'
+    else:
+        return "high"
+
+
+def update_record(station, weather):
+    print(f"Updating {station['address']}")
+    if "/" in station['address']:
+        data = pd.read_csv('gs://dbikes-planner.appspot.com/station_records/Princes Street.csv')
+    else:
+        data = pd.read_csv(f"gs://dbikes-planner.appspot.com/station_records/{station['address']}.csv")
+
+    epoch_time = station['last_update']
+
+    entry_datetime = datetime.fromtimestamp(epoch_time / 1000)
+
+    if is_time_between(time(3, 30), time(5, 0), entry_datetime.time()):
+        return
+
+    last_line = data.tail(3).to_csv()
+
+    if entry_datetime.isoformat() in last_line:
+        return
+
+    day_index = entry_datetime.weekday()
+
+    if day_index <= 4:
+        day_type = 0
+    else:
+        day_type = 10
+
+    new_row = {'available_bikes': station['available_bikes'],
+               'available_bike_stands': station['available_bike_stands'],
+               'time_of_day': datetime_to_seconds(entry_datetime), 'type_of_day': day_type,
+               'day_of_year': entry_datetime.timetuple().tm_yday, 'iso_date': entry_datetime.isoformat(),
+               'temperature': weather['temperature'], 'relative_humidity': weather['humidity'],
+               'wind_speed': weather['wind_speed'], 'rain': weather['rain'], 'visibility': weather['visibility'],
+               'bike_availability': category(station['available_bikes']),
+               'bike_stand_availability': category(station['available_bike_stands']),
+               'unix_timestamp': entry_datetime.timestamp() // 3600}
+
+    new_row_dataframe = pd.DataFrame(new_row, index=[0])
+
+    combined_df = pd.concat([data, new_row_dataframe], ignore_index=True)
+
+    gcs.Bucket('dbikes-planner.appspot.com').item(f'station_records/{station["address"]}.csv') \
+        .write_to(combined_df.to_csv(index=False), 'text/csv')

webapp/flex_app.yaml

@@ -0,0 +1,13 @@
+runtime: python
+env: flex
+entrypoint: gunicorn -b :$PORT main:app
+
+runtime_config:
+  python_version: 3
+
+manual_scaling:
+  instances: 1
+resources:
+  cpu: 1
+  memory_gb: 0.5
+  disk_size_gb: 10

webapp/routes/__init__.py

@@ -5,3 +5,4 @@
 from .prediction_service import *
 from .history_service import *
 from .feedback_service import *
+from .health_check_service import *

webapp/routes/health_check_service.py

@@ -0,0 +1,9 @@
+from . import routes
+
+from flask import request
+from google.cloud import datastore
+
+
+@routes.route('/health-check')
+def health_check():
+    return 'OK!'

webapp/routes/prediction_service.py

@@ -1,12 +1,10 @@
 from datetime import datetime, timedelta
 from . import routes
 
-import update_station_records
 from flask import request
 import json
 import pandas as pd
 from sklearn.neighbors import KNeighborsClassifier
-from apscheduler.schedulers.background import BackgroundScheduler
 import requests
 
 station_names = ['Smithfield North', 'Parnell Square North', 'Clonmel Street', 'Avondale Road', 'Mount Street Lower',
@@ -34,8 +32,6 @@
                  'Grangegorman Lower (South)', 'Mountjoy Square West', 'Wilton Terrace', 'Emmet Road',
                  'Heuston Bridge (North)', 'Leinster Street South', 'Blackhall Place']
 
-weather = {}
-
 
 def update_weather():
     print("Updating weather data...")
@@ -59,21 +55,9 @@ def update_weather():
     else:
         parsed_weather['visibility'] = 0
 
-    global weather
-    weather = parsed_weather
-
     print("Weather data update complete!")
 
-
-def update_bike_data(current_weather):
-    result = pd.read_json(
-        "https://api.jcdecaux.com/vls/v1/stations?contract=dublin&apiKey=6e5c2a98e60a3336ecaede8f8c8688da25144692")
-
-
-    print("Updating bike data...")
-    for _index, row in result.iterrows():
-        update_station_records.update_record(row, current_weather)
-    print("Bike data refresh complete!")
+    return parsed_weather
 
 
 def read_file_for_station(station_name):
@@ -106,19 +90,6 @@ def get_fitted_bikes_model(station_name):
     return model
 
 
-def refresh_data():
-    update_weather()
-    if datetime.now().hour >= 5:
-        update_bike_data(weather)
-
-
-refresh_data()
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(func=refresh_data, trigger="interval", minutes=5)
-scheduler.start()
-
-
 def predict_availability(station, minutes, type):
     current_time = datetime.now()
     request_time = current_time + timedelta(minutes=minutes)
@@ -132,6 +103,8 @@ def predict_availability(station, minutes, type):
     else:
         model = get_fitted_bikestands_model(station)
 
+    weather = update_weather()
+
     prediction = model.predict([[time_of_day, type_of_day, day_of_year, weather['temperature'],
                                  weather['humidity'], weather['wind_speed'], weather['rain'],
                                  weather['visibility'], current_time.timestamp()//3600]])
@@ -175,5 +148,5 @@ def predict_bike_stands_availability(journey_type):
 
 
 @routes.errorhandler(404)
-def page_not_found(e):
+def page_not_found():
     return "<h1>404</h1><p>The resource could not be found. Sorry!</p>", 404