diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 58e43b0..64634b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,7 +9,8 @@ on: # Let's set all project specific definitions globally env: PRJ_NAME: LiveTraffic # The plugin's name, expected to be the .xpl file's name and used as the plugin folder name - version_beta: ${{ github.ref != 'refs/heads/master' && '1' || '0' }} # On non-master branches we build BETA versions + # On master branches or when explicitely tagged we build prod versions, otherwise BETA + version_beta: ${{ (github.ref != 'refs/heads/master' && github.ref_type != 'tag') && '1' || '0' }} jobs: ##################################### diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d5bfd9..28d1bad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ else() endif() project(LiveTraffic - VERSION 3.5.0 + VERSION 3.5.1 DESCRIPTION "LiveTraffic X-Plane plugin") # Building a Beta version can be demanded via ENV variable 'version_beta' being set to 1 diff --git a/Data/RealTraffic/RT_API.sjson/729379088.21102 b/Data/RealTraffic/RT_API.sjson/729379088.21102 new file mode 100644 index 0000000..802e396 Binary files /dev/null and b/Data/RealTraffic/RT_API.sjson/729379088.21102 differ diff --git a/Data/RealTraffic/RT_API.sjson/729379133.463946 b/Data/RealTraffic/RT_API.sjson/729379133.463946 new file mode 100644 index 0000000..2fc3cec Binary files /dev/null and b/Data/RealTraffic/RT_API.sjson/729379133.463946 differ diff --git a/Data/RealTraffic/RT_API.sjson/data b/Data/RealTraffic/RT_API.sjson/data index e0e0b0f..d22ac3b 100644 Binary files a/Data/RealTraffic/RT_API.sjson/data and b/Data/RealTraffic/RT_API.sjson/data differ diff --git a/Data/RealTraffic/RT_API.sjson/metaData b/Data/RealTraffic/RT_API.sjson/metaData index 7969afc..2c9a101 100644 Binary files a/Data/RealTraffic/RT_API.sjson/metaData and b/Data/RealTraffic/RT_API.sjson/metaData differ diff --git a/Include/Constants.h b/Include/Constants.h index 26624b7..1c9b894 100644 --- a/Include/Constants.h +++ b/Include/Constants.h @@ -228,6 +228,7 @@ constexpr const char* REMOTE_SIGNATURE = "TwinFan.plugin.XPMP2.Remote"; #define MENU_HAVE_TCAS "TCAS controlled" #define MENU_HAVE_TCAS_REQUSTD "TCAS controlled (requested)" #define MENU_TOGGLE_LABELS "Labels shown" +#define MENU_TOGGLE_AC_AHEAD "Hide Aircraft ahead" #define MENU_SETTINGS_UI "Settings..." #define MENU_HELP "Help" #define MENU_HELP_DOCUMENTATION "Documentation" @@ -237,6 +238,8 @@ constexpr const char* REMOTE_SIGNATURE = "TwinFan.plugin.XPMP2.Remote"; #define MENU_HELP_AC_INFO_WND "A/C Info Window" #define MENU_HELP_SETTINGS "Settings" #define MENU_HELP_INSTALL_CSL "Installaton of CSL Models" +#define MENU_HELP_SUPPORT_FORUM "Support Forum" +#define MENU_HELP_SUPPORT_HOWTO "Support HowTo" #define MENU_NEWVER "New Version %s available!" #ifdef DEBUG #define MENU_RELOAD_PLUGINS "Reload all Plugins (Caution!)" @@ -270,6 +273,9 @@ constexpr const char* REMOTE_SIGNATURE = "TwinFan.plugin.XPMP2.Remote"; #define HELP_SET_CSL "setup/configuration/settings-csl" #define HELP_SET_DEBUG "setup/configuration/settings-debug" +#define URL_SUPPORT_FORUM "https://forums.x-plane.org/index.php?/forums/forum/457-livetraffic-support/" +#define URL_SUPPORT_HOWTO "https://forums.x-plane.org/index.php?/forums/topic/174691-support" + //MARK: File Paths // these are under the plugins directory #define PATH_FLIGHT_MODELS "Resources/FlightModels.prf" diff --git a/Include/DataRefs.h b/Include/DataRefs.h index 33a0039..63b9511 100644 --- a/Include/DataRefs.h +++ b/Include/DataRefs.h @@ -449,6 +449,7 @@ enum cmdRefsLT { CR_AC_DISPLAYED, CR_AC_TCAS_CONTROLLED, CR_LABELS_TOGGLE, + CR_TOGGLE_AC_AHEAD, ///< Toggle visibility of the a/c ahead CR_SETTINGS_UI, CNT_CMDREFS_LT // always last, number of elements }; @@ -1036,9 +1037,9 @@ class DataRefs int DecNumAc(); // Get XP System Path - inline std::string GetXPSystemPath() const { return XPSystemPath; } - inline std::string GetLTPluginPath() const { return LTPluginPath; } - inline std::string GetDirSeparator() const { return DirSeparator; } + const std::string& GetXPSystemPath() const { return XPSystemPath; } + const std::string& GetLTPluginPath() const { return LTPluginPath; } + const std::string& GetDirSeparator() const { return DirSeparator; } // Load/save config file (basically a subset of LT dataRefs) bool LoadConfigFile(); diff --git a/Include/LTOpenSky.h b/Include/LTOpenSky.h index 392c7c2..43a122b 100644 --- a/Include/LTOpenSky.h +++ b/Include/LTOpenSky.h @@ -103,6 +103,7 @@ constexpr size_t OPSKY_MD_TEXT_VEHICLE_LEN = 20; ///< length after which cate #define OPSKY_MD_DB_NAME "OpenSky Masterdata File" #define OPSKY_MD_DB_URL "https://opensky-network.org/datasets/metadata/" +#define OPSKY_MD_DB_FILE_BEGIN "aircraft-database-complete-" #define OPSKY_MD_DB_FILE "aircraft-database-complete-%04d-%02d.csv" #define OPSKY_ROUTE_URL "https://opensky-network.org/api/routes?callsign=" diff --git a/Include/LTRealTraffic.h b/Include/LTRealTraffic.h index 48d9585..c124047 100644 --- a/Include/LTRealTraffic.h +++ b/Include/LTRealTraffic.h @@ -79,8 +79,9 @@ constexpr double RT_VSI_AIRBORNE = 80.0; ///< if VSI is more than this then w constexpr long RT_DRCT_DEFAULT_WAIT = 8000L; ///< [ms] Default wait time between traffic requests constexpr std::chrono::seconds RT_DRCT_ERR_WAIT = std::chrono::seconds(5); ///< standard wait between errors constexpr std::chrono::minutes RT_DRCT_WX_WAIT = std::chrono::minutes(10); ///< How often to update weather? +constexpr std::chrono::seconds RT_DRCT_WX_ERR_WAIT = std::chrono::seconds(60); ///< How long to wait after receiving an weather error? constexpr long RT_DRCT_WX_DIST = 10L * M_per_NM; ///< Distance for which weather is considered valid, greater than that and we re-request -constexpr int RT_DRCT_MAX_WX_ERR = 10; ///< Max number of consecutive errors during weather requests we wait for...before not asking for weather any longer +constexpr int RT_DRCT_MAX_WX_ERR = 5; ///< Max number of consecutive errors during initial weather requests we wait for...before not asking for weather any longer /// Fields in a response of a direct connection's request enum RT_DIRECT_FIELDS_TY { @@ -265,12 +266,13 @@ class RealTrafficConnection : public LTFlightDataChannel /// Weather data struct WxTy { double QNH = NAN; ///< baro pressure - std::chrono::steady_clock::time_point time; ///< time when RealTraffic weather was received + std::chrono::steady_clock::time_point next; ///< next time to request RealTraffic weather positionTy pos; ///< viewer position for which we received Realtraffic weather long tOff = 0; ///< time offset for which we requested weather int nErr = 0; ///< How many errors did we have during weather requests? - WxTy& operator = (const CurrTy& o); ///< fill from `current` data + /// Set all relevant values + void set (double qnh, const CurrTy& o, bool bResetErr = true); } rtWx; ///< Data with which latest weather was requested /// How many flights does RealTraffic have in total? long lTotalFlights = -1; diff --git a/Include/LiveTraffic.h b/Include/LiveTraffic.h index 3bf367d..4142085 100644 --- a/Include/LiveTraffic.h +++ b/Include/LiveTraffic.h @@ -220,6 +220,10 @@ bool RemoteFileDownload (const std::string& url, const std::string& path); std::string& str_toupper(std::string& s); /// return a std::string copy converted to uppercase std::string str_toupper_c(const std::string& s); +/// Case-insensitive equal +bool striequal (const std::string& a, const std::string& b); +/// Case-insensitive begins with +bool stribeginwith (const std::string& s, const std::string& begin); // are all chars alphanumeric? bool str_isalnum(const std::string& s); // limits text to m characters, replacing the last ones with ... if too long diff --git a/LiveTraffic.xcodeproj/project.pbxproj b/LiveTraffic.xcodeproj/project.pbxproj index bd30951..af73961 100755 --- a/LiveTraffic.xcodeproj/project.pbxproj +++ b/LiveTraffic.xcodeproj/project.pbxproj @@ -790,7 +790,7 @@ LIVETRAFFIC_VERSION_BETA = 0; LIVETRAFFIC_VER_MAJOR = 3; LIVETRAFFIC_VER_MINOR = 5; - LIVETRAFFIC_VER_PATCH = 0; + LIVETRAFFIC_VER_PATCH = 1; LLVM_LTO = NO; MACH_O_TYPE = mh_dylib; MACOSX_DEPLOYMENT_TARGET = 10.15; @@ -896,7 +896,7 @@ LIVETRAFFIC_VERSION_BETA = 0; LIVETRAFFIC_VER_MAJOR = 3; LIVETRAFFIC_VER_MINOR = 5; - LIVETRAFFIC_VER_PATCH = 0; + LIVETRAFFIC_VER_PATCH = 1; LLVM_LTO = YES; MACH_O_TYPE = mh_dylib; MACOSX_DEPLOYMENT_TARGET = 10.15; diff --git a/Src/DataRefs.cpp b/Src/DataRefs.cpp index 2ebb8a2..3e5083c 100644 --- a/Src/DataRefs.cpp +++ b/Src/DataRefs.cpp @@ -684,6 +684,7 @@ struct cmdRefDescrTy { {"LiveTraffic/Aircrafts/Display", "Starts/Stops display of live aircraft"}, {"LiveTraffic/Aircrafts/TCAS_Control", "TCAS Control toggle: Tries to take control over AI aircraft, or release it"}, {"LiveTraffic/Aircrafts/Toggle_Labels", "Toggle display of labels in current view"}, + {"LiveTraffic/Aircrafts/Toggle_Ahead", "Toggle visibility of aircraft ahead"}, {"LiveTraffic/Settings/Open", "Opens/Closes the Settings window"}, }; diff --git a/Src/LTChannel.cpp b/Src/LTChannel.cpp index 93f7ba2..d7ac8c3 100644 --- a/Src/LTChannel.cpp +++ b/Src/LTChannel.cpp @@ -887,8 +887,12 @@ void LTFlightDataDisable() void LTFlightDataStop() { + /// @see https://github.com/Homebrew/homebrew-core/issues/158759#issuecomment-1874091015 + /// To be able to reload plugins we don't properly call global cleanup +#if not APL // cleanup global CURL stuff curl_global_cleanup(); +#endif } // diff --git a/Src/LTMain.cpp b/Src/LTMain.cpp index e26bf77..add3afb 100644 --- a/Src/LTMain.cpp +++ b/Src/LTMain.cpp @@ -387,6 +387,25 @@ std::string str_toupper_c(const std::string& s) return c; } +// Case-insensitive equal +/// @see https://stackoverflow.com/a/4119881 +bool striequal (const std::string& a, const std::string& b) +{ + return std::equal(a.begin(), a.end(), b.begin(), b.end(), + [](unsigned char x, unsigned char y) + { return std::tolower(x) == std::tolower(y); }); +} + +// Case-insensitive begins with +bool stribeginwith (const std::string& s, const std::string& begin) +{ + if (begin.size() > s.size()) return false; + return std::equal(begin.begin(), begin.end(), s.begin(), + [](unsigned char x, unsigned char y) + { return std::tolower(x) == std::tolower(y); }); +} + + bool str_isalnum(const std::string& s) { return std::all_of(s.cbegin(), s.cend(), [](unsigned char c){return isalnum(c);}); diff --git a/Src/LTOpenSky.cpp b/Src/LTOpenSky.cpp index 70c3343..b093a6e 100644 --- a/Src/LTOpenSky.cpp +++ b/Src/LTOpenSky.cpp @@ -1,928 +1,972 @@ -/// @file LTOpenSky.cpp -/// @brief OpenSky Network: Requests and processes live tracking and aircraft master data -/// @see https://opensky-network.org/ -/// @details Implements OpenSkyConnection and OpenSkyAcMasterdata:\n -/// - Provides a proper REST-conform URL\n -/// - Interprets the response and passes the tracking data on to LTFlightData.\n -/// @author Birger Hoppe -/// @copyright (c) 2018-2020 Birger Hoppe -/// @copyright Permission is hereby granted, free of charge, to any person obtaining a -/// copy of this software and associated documentation files (the "Software"), -/// to deal in the Software without restriction, including without limitation -/// the rights to use, copy, modify, merge, publish, distribute, sublicense, -/// and/or sell copies of the Software, and to permit persons to whom the -/// Software is furnished to do so, subject to the following conditions:\n -/// The above copyright notice and this permission notice shall be included in -/// all copies or substantial portions of the Software.\n -/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -/// THE SOFTWARE. - -// All includes are collected in one header -#include "LiveTraffic.h" - -// -//MARK: OpenSky -// - -// Constructor -OpenSkyConnection::OpenSkyConnection () : -LTFlightDataChannel(DR_CHANNEL_OPEN_SKY_ONLINE, OPSKY_NAME) -{ - // purely informational - urlName = OPSKY_CHECK_NAME; - urlLink = OPSKY_CHECK_URL; - urlPopup = OPSKY_CHECK_POPUP; -} - -// virtual thread main function -void OpenSkyConnection::Main () -{ - // This is a communication thread's main function, set thread's name and C locale - ThreadSettings TS ("LT_OpSky", LC_ALL_MASK); - - while ( shallRun() ) { - // LiveTraffic Top Level Exception Handling - try { - // basis for determining when to be called next - tNextWakeup = std::chrono::steady_clock::now(); - - // where are we right now? - const positionTy pos (dataRefs.GetViewPos()); - - // If the camera position is valid we can request data around it - if (pos.isNormal()) { - // Next wakeup is "refresh interval" from _now_ - tNextWakeup += std::chrono::seconds(dataRefs.GetFdRefreshIntvl()); - - // fetch data and process it - if (FetchAllData(pos) && ProcessFetchedData()) - // reduce error count if processed successfully - // as a chance to appear OK in the long run - DecErrCnt(); - } - else { - // Camera position is yet invalid, retry in a second - tNextWakeup += std::chrono::seconds(1); - } - - // sleep for FD_REFRESH_INTVL or if woken up for termination - // by condition variable trigger - { - std::unique_lock lk(FDThreadSynchMutex); - FDThreadSynchCV.wait_until(lk, tNextWakeup, - [this]{return !shallRun();}); - } - - } catch (const std::exception& e) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); - IncErrCnt(); - } catch (...) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); - IncErrCnt(); - } - } -} - - -// Initialize CURL, adding OpenSky credentials -bool OpenSkyConnection::InitCurl () -{ - // Standard-init first (repeated call will just return true without effect) - if (!LTOnlineChannel::InitCurl()) - return false; - - // if there are credentials then now is the moment to add them - std::string usr, pwd; - dataRefs.GetOpenSkyCredentials(usr, pwd); - if (!usr.empty() && !pwd.empty()) { - curl_easy_setopt(pCurl, CURLOPT_USERNAME, usr.data()); - curl_easy_setopt(pCurl, CURLOPT_PASSWORD, pwd.data()); - } else { - curl_easy_setopt(pCurl, CURLOPT_USERNAME, nullptr); - curl_easy_setopt(pCurl, CURLOPT_PASSWORD, nullptr); - } - - // read headers (for remaining requests info) - curl_easy_setopt(pCurl, CURLOPT_HEADERFUNCTION, ReceiveHeader); - - return true; -} - -// read header and parse for request remaining -size_t OpenSkyConnection::ReceiveHeader(char *buffer, size_t size, size_t nitems, void *) -{ - const size_t len = nitems * size; - static size_t lenRRemain = strlen(OPSKY_RREMAIN); - static size_t lenRetry = strlen(OPSKY_RETRY); - char num[50]; - - // Remaining? - if (len > lenRRemain && - memcmp(buffer, OPSKY_RREMAIN, lenRRemain) == 0) - { - const size_t copyCnt = std::min(len-lenRRemain,sizeof(num)-1); - memcpy(num, buffer+lenRRemain, copyCnt); - num[copyCnt]=0; // zero termination - - long rRemain = std::atol(num); - // Issue a warning when coming close to the end - if (rRemain != dataRefs.OpenSkyRRemain) { - if (rRemain == 50 || rRemain == 10) { - SHOW_MSG(logWARN, "OpenSky: Only %ld requests left today for ca. %ld minutes of data", - rRemain, - (rRemain * dataRefs.GetFdRefreshIntvl()) / 60); - } - dataRefs.OpenSkyRRemain = rRemain; - if (rRemain > 0) - dataRefs.OpenSkyRetryAt.clear(); - } - } - // Retry-after-seconds? - else if (len > lenRetry && - memcmp(buffer, OPSKY_RETRY, lenRetry) == 0) - { - const size_t copyCnt = std::min(len-lenRetry,sizeof(num)-1); - memcpy(num, buffer+lenRetry, copyCnt); - num[copyCnt]=0; // zero termination - long secRetry = std::atol(num); // seconds till retry - // convert that to a local timestamp for the user to use - const std::time_t tRetry = std::time(nullptr) + secRetry; - std::strftime(num, sizeof(num), "%d-%b %H:%M", std::localtime(&tRetry)); - dataRefs.OpenSkyRetryAt = num; - dataRefs.OpenSkyRRemain = 0; - } - - // always say we processed everything, otherwise HTTP processing would stop! - return len; -} - -// put together the URL to fetch based on current view position -std::string OpenSkyConnection::GetURL (const positionTy& pos) -{ - // we add 10% to the bounding box to have some data ready once the plane is close enough for display - boundingBoxTy box (pos, double(dataRefs.GetFdStdDistance_m()) * 1.10); - char url[128] = ""; - snprintf(url, sizeof(url), - OPSKY_URL_ALL, - box.se.lat(), // lamin - box.nw.lon(), // lomin - box.nw.lat(), // lamax - box.se.lon() ); // lomax - return std::string(url); -} - -// update shared flight data structures with received flight data -// "a4d85d","UJC11 ","United States",1657226901,1657226901,-90.2035,38.8157,2758.44,false,128.1,269.54,-6.5,null,2895.6,"4102",false,0 -bool OpenSkyConnection::ProcessFetchedData () -{ - char buf[100]; - - // any a/c filter defined for debugging purposes? - std::string acFilter ( dataRefs.GetDebugAcFilter() ); - - // data is expected to be in netData string - // short-cut if there is nothing - if ( !netDataPos ) return true; - - // Only proceed in case HTTP response was OK - if (httpResponse != HTTP_OK) { - // Unauthorized? - if (httpResponse == HTTP_UNAUTHORIZED) { - SHOW_MSG(logERR, "OpenSky: Unauthorized! Verify username/password in settings."); - SetValid(false,false); - SetEnable(false); // also disable to directly allow user/pwd change...and won't work on retry anyway - return false; - } - - // Ran out of requests? - if (httpResponse == HTTP_TOO_MANY_REQU) { - SHOW_MSG(logERR, "OpenSky: Used up request credit for today, try again on %s", - dataRefs.OpenSkyRetryAt.empty() ? "" : dataRefs.OpenSkyRetryAt.c_str()); - SetValid(false,false); - return false; - } - - // Timeouts are so common recently with OpenSky that we no longer treat them as errors, - // but we inform the user every once in a while - if (httpResponse == HTTP_GATEWAY_TIMEOUT && - httpResponse == HTTP_TIMEOUT) - { - static std::chrono::time_point lastTimeoutWarn; - auto tNow = std::chrono::steady_clock::now(); - if (tNow > lastTimeoutWarn + std::chrono::minutes(5)) { - lastTimeoutWarn = tNow; - SHOW_MSG(logWARN, "%s communication unreliable due to timeouts!", pszChName); - } - } - else { // anything else is serious - IncErrCnt(); - } - return false; - } - - // now try to interpret it as JSON - JSONRootPtr pRoot (netData); - if (!pRoot) { LOG_MSG(logERR,ERR_JSON_PARSE); IncErrCnt(); return false; } - - // let's cycle the aircraft - // first get the structre's main object - JSON_Object* pObj = json_object(pRoot.get()); - if (!pObj) { LOG_MSG(logERR,ERR_JSON_MAIN_OBJECT); IncErrCnt(); return false; } - - // for determining an offset as compared to network time we need to know network time - double opSkyTime = jog_n(pObj, OPSKY_TIME); - if (opSkyTime > JAN_FIRST_2019) - // if reasonable add this to our time offset calculation - dataRefs.ChTsOffsetAdd(opSkyTime); - - // Cut-off time: We ignore tracking data, which is "in the past" compared to simTime - const double tsCutOff = dataRefs.GetSimTime(); - - // We need to calculate distance to current camera later on - const positionTy viewPos = dataRefs.GetViewPos(); - - // fetch the aircraft array - JSON_Array* pJAcList = json_object_get_array(pObj, OPSKY_AIRCRAFT_ARR); - if (!pJAcList) { - // a/c array not found: can just mean it is 'null' as in - // the empty result set: {"time":1541978120,"states":null} - JSON_Value* pJSONVal = json_object_get_value(pObj, OPSKY_AIRCRAFT_ARR); - if (!pJSONVal || json_type(pJSONVal) != JSONNull) { - // well...it is something else, so it is malformed, bail out - LOG_MSG(logERR,ERR_JSON_ACLIST,OPSKY_AIRCRAFT_ARR); - IncErrCnt(); - return false; - } - } - // iterate all aircraft in the received flight data (can be 0) - else for ( size_t i=0; i < json_array_get_count(pJAcList); i++ ) - { - // get the aircraft (which is just an array of values) - JSON_Array* pJAc = json_array_get_array(pJAcList,i); - if (!pJAc) { - LOG_MSG(logERR,ERR_JSON_AC,i+1,OPSKY_AIRCRAFT_ARR); - if (IncErrCnt()) - continue; - else - return false; - } - - // the key: transponder Icao code - LTFlightData::FDKeyTy fdKey (LTFlightData::KEY_ICAO, - jag_s(pJAc, OPSKY_TRANSP_ICAO)); - - // not matching a/c filter? -> skip it - if ((!acFilter.empty() && (fdKey != acFilter)) ) - { - continue; - } - - // position time - const double posTime = jag_n(pJAc, OPSKY_POS_TIME); - if (posTime <= tsCutOff) - continue; - - try { - // from here on access to fdMap guarded by a mutex - // until FD object is inserted and updated - std::unique_lock mapFdLock (mapFdMutex); - - // Check for duplicates with OGN/FLARM, potentially replaces the key type - LTFlightData::CheckDupKey(fdKey, LTFlightData::KEY_FLARM); - - // get the fd object from the map, key is the transpIcao - // this fetches an existing or, if not existing, creates a new one - LTFlightData& fd = mapFd[fdKey]; - - // also get the data access lock once and for all - // so following fetch/update calls only make quick recursive calls - std::lock_guard fdLock (fd.dataAccessMutex); - // now that we have the detail lock we can release the global one - mapFdLock.unlock(); - - // completely new? fill key fields - if ( fd.empty() ) - fd.SetKey(fdKey); - - // fill static data - LTFlightData::FDStaticData stat; - stat.country = jag_s(pJAc, OPSKY_COUNTRY); - stat.call = jag_s(pJAc, OPSKY_CALL); - while (!stat.call.empty() && stat.call.back() == ' ') // trim trailing spaces - stat.call.pop_back(); - if (!fdKey.empty()) { - snprintf(buf, sizeof(buf), OPSKY_SLUG_FMT, fdKey.num); - stat.slug = buf; - } - - // dynamic data - { // unconditional...block is only for limiting local variables - LTFlightData::FDDynamicData dyn; - - // non-positional dynamic data - dyn.radar.code = (long)jag_sn(pJAc, OPSKY_RADAR_CODE); - dyn.gnd = jag_b(pJAc, OPSKY_GND); - dyn.heading = jag_n_nan(pJAc, OPSKY_HEADING); - dyn.spd = jag_n(pJAc, OPSKY_SPD); - dyn.vsi = jag_n(pJAc, OPSKY_VSI); - dyn.ts = posTime; - dyn.pChannel = this; - - // position - const double baroAlt_m = jag_n_nan(pJAc, OPSKY_BARO_ALT); - const double geoAlt_m = BaroAltToGeoAlt_m(baroAlt_m, dataRefs.GetPressureHPA()); - positionTy pos (jag_n_nan(pJAc, OPSKY_LAT), - jag_n_nan(pJAc, OPSKY_LON), - geoAlt_m, - posTime, - dyn.heading); - pos.f.onGrnd = dyn.gnd ? GND_ON : GND_OFF; - - // Update static data - fd.UpdateData(std::move(stat), pos.dist(viewPos)); - - // position is rather important, we check for validity - // (we do allow alt=NAN if on ground as this is what OpenSky returns) - if ( pos.isNormal(true) ) - fd.AddDynData(dyn, 0, 0, &pos); - else - LOG_MSG(logDEBUG,ERR_POS_UNNORMAL,fdKey.c_str(),pos.dbgTxt().c_str()); - } - } catch(const std::system_error& e) { - LOG_MSG(logERR, ERR_LOCK_ERROR, "mapFd", e.what()); - } - } - - // success - return true; -} - - -// get status info, including remaining requests -std::string OpenSkyConnection::GetStatusText () const -{ - std::string s = LTChannel::GetStatusText(); - if (IsValid()) { - if (dataRefs.OpenSkyRRemain < LONG_MAX) - { - s += " | "; - s += std::to_string(dataRefs.OpenSkyRRemain); - s += " requests left today"; - } - } else { - if (!dataRefs.OpenSkyRetryAt.empty()) - { - s += ", retry at "; - s += dataRefs.OpenSkyRetryAt; - } - } - - return s; -} - - - -// -//MARK: OpenSkyAcMasterdata -// - -// Constructor -OpenSkyAcMasterdata::OpenSkyAcMasterdata () : -LTACMasterdataChannel(DR_CHANNEL_OPEN_SKY_AC_MASTERDATA, OPSKY_MD_NAME) -{ - // purely informational - urlName = OPSKY_MD_CHECK_NAME; - urlLink = OPSKY_MD_CHECK_URL; - urlPopup = OPSKY_MD_CHECK_POPUP; -} - -// accept requests that aren't in the ignore lists -bool OpenSkyAcMasterdata::AcceptRequest (const acStatUpdateTy& r) -{ - if ((r.type == DATREQU_ROUTE || // accepting all kinds of call signs - r.acKey.eKeyType == LTFlightData::KEY_ICAO) && // but only ICAO-typed master data requests - !ShallIgnore(r)) - { - InsertRequest(r); - return true; - } - return false; -} - -// virtual thread main function -void OpenSkyAcMasterdata::Main () -{ - // This is a communication thread's main function, set thread's name and C locale - ThreadSettings TS ("LT_OpSkyMaster", LC_ALL_MASK); - RegisterMasterDataChn(this); // Register myself as a master data channel - tSetRequCleared = dataRefs.GetMiscNetwTime(); - - while ( shallRun() ) { - // LiveTraffic Top Level Exception Handling - try { - // if there is something to request, fetch the data and process it - if (FetchNextRequest()) - { - if (FetchAllData(positionTy()) && ProcessFetchedData()) { - // reduce error count if processed successfully, as a chance to appear OK in the long run - DecErrCnt(); - } else { - // Could not find the data here, pass on the request - PassOnRequest(this, currRequ); - } - } - - // We must wait a moment between any two requests just not to overload the server - std::this_thread::sleep_for(OPSKY_WAIT_BETWEEN); - - // sleep a bit or until woken up for termination by condition variable trigger - if (!HaveAnyRequest()) - { - std::unique_lock lk(FDThreadSynchMutex); - FDThreadSynchCV.wait_for(lk, OPSKY_WAIT_NOQUEUE, - [this]{return !shallRun() || HaveAnyRequest();}); - } - - // Every 3s clear up outdated requests waiting in queue - if (CheckEverySoOften(tSetRequCleared, 3.0f)) - MaintainMasterDataRequests(); - - } catch (const std::exception& e) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); - IncErrCnt(); - } catch (...) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); - IncErrCnt(); - } - } - - // Before leaving clear the request queue - std::unique_lock lock (mtxMaster); - if (bFDMainStop) // in case of all stopping just throw them away - setAcStatRequ.clear(); - else - MaintainMasterDataRequests(); // otherwise clear up and pass on - UnregisterMasterDataChn(this); // Unregister myself as a master data channel -} - - -// Returns the master data or route URL to query -std::string OpenSkyAcMasterdata::GetURL (const positionTy& /*pos*/) -{ - switch (currRequ.type) { - case DATREQU_AC_MASTER: - return std::string(OPSKY_MD_URL) + URLEncode(currRequ.acKey.key); - case DATREQU_ROUTE: - return std::string(OPSKY_ROUTE_URL) + URLEncode(currRequ.callSign); - case DATREQU_NONE: - return std::string(); - } - return std::string(); -} - -// process each master data line read from OpenSky -bool OpenSkyAcMasterdata::ProcessFetchedData () -{ - // If the requested data just wasn't found add it to the ignore list - if (httpResponse == HTTP_NOT_FOUND) { - AddIgnore(); - return false; - } - // Any other result or no message is technically not OK - else if (httpResponse != HTTP_OK || !netDataPos) { - IncErrCnt(); - return false; - } - - // Try to interpret is as JSON and get the main JSON object - JSONRootPtr pRoot (netData); - if (!pRoot) { LOG_MSG(logERR,ERR_JSON_PARSE); IncErrCnt(); return false; } - JSON_Object* pObj = json_object(pRoot.get()); - if (!pObj) { LOG_MSG(logERR,ERR_JSON_MAIN_OBJECT); IncErrCnt(); return false; } - - // Pass on the further processing depending on the request type - switch (currRequ.type) { - case DATREQU_AC_MASTER: - return ProcessMasterData(pObj); - case DATREQU_ROUTE: - return ProcessRouteInfo(pObj); - case DATREQU_NONE: - break; - } - return false; -} - - -// Process received aircraft master data -bool OpenSkyAcMasterdata::ProcessMasterData (JSON_Object* pJAc) -{ - LTFlightData::FDKeyTy fdKey; // the key: transponder Icao code, filled from response! - LTFlightData::FDStaticData statDat; // here we collect the master data - - // fetch values from the online data - fdKey.SetKey(LTFlightData::KEY_ICAO, - jog_s(pJAc, OPSKY_MD_TRANSP_ICAO)); - statDat.reg = jog_s(pJAc, OPSKY_MD_REG); - statDat.country = jog_s(pJAc, OPSKY_MD_COUNTRY); - statDat.acTypeIcao = jog_s(pJAc, OPSKY_MD_AC_TYPE_ICAO); - statDat.man = jog_s(pJAc, OPSKY_MD_MAN); - statDat.mdl = jog_s(pJAc, OPSKY_MD_MDL); - statDat.catDescr = jog_s(pJAc, OPSKY_MD_CAT_DESCR); - statDat.op = jog_s(pJAc, OPSKY_MD_OP); - statDat.opIcao = jog_s(pJAc, OPSKY_MD_OP_ICAO); - - // -- Ground vehicle identification -- - // OpenSky only delivers "category description" and has a - // pretty clear indicator for a ground vehicle - if (statDat.acTypeIcao.empty() && // don't know a/c type yet - (statDat.catDescr.find(OPSKY_MD_TEXT_VEHICLE) != std::string::npos || - // I'm having the feeling that if nearly all is empty and the category description is "No Info" then it's often also a ground vehicle - (statDat.catDescr.find(OPSKY_MD_TEXT_NO_CAT) != std::string::npos && - statDat.man.empty() && - statDat.mdl.empty() && - statDat.opIcao.empty()))) - { - // we assume ground vehicle - statDat.acTypeIcao = dataRefs.GetDefaultCarIcaoType(); - // The category description usually is something like - // "Surface Vehicle – Service Vehicle" - // Save the latter part if we have no model info yet - if (statDat.mdl.empty() && - statDat.catDescr.find(OPSKY_MD_TEXT_VEHICLE) != std::string::npos && - statDat.catDescr.length() > OPSKY_MD_TEXT_VEHICLE_LEN) - { - statDat.mdl = statDat.catDescr.c_str() + OPSKY_MD_TEXT_VEHICLE_LEN; - } - } - // Replace type GRND with our default car type, too - else if (statDat.acTypeIcao == "GRND" || statDat.acTypeIcao == "GND") - statDat.acTypeIcao = dataRefs.GetDefaultCarIcaoType(); - - // Perform the update - UpdateStaticData(fdKey, statDat); - return true; -} - - -// Process received route info -bool OpenSkyAcMasterdata::ProcessRouteInfo (JSON_Object* pJRoute) -{ - LTFlightData::FDStaticData statDat; // here we collect the master data - - // fetch values from the online data - // route is an array of typically 2 entries, but can have more - // "route":["EDDM","LIMC"] - JSON_Array* pJRArr = json_object_get_array(pJRoute, OPSKY_ROUTE_ROUTE); - if (pJRArr) { - size_t cnt = json_array_get_count(pJRArr); - for (size_t i = 0; i < cnt; i++) - statDat.stops.push_back(jag_s(pJRArr, i)); - } - - // flight number: made up of IATA and actual number - statDat.flight = jog_s(pJRoute,OPSKY_ROUTE_OP_IATA); - double flightNr = jog_n_nan(pJRoute,OPSKY_ROUTE_FLIGHT_NR); - if (!std::isnan(flightNr)) - statDat.flight += std::to_string(lround(flightNr)); - - // update the a/c's master data - UpdateStaticData(currRequ.acKey, statDat); - return true; -} - -// -// MARK: OpenSky Master Data File -// - - -// Constructor -OpenSkyAcMasterFile::OpenSkyAcMasterFile () : -LTACMasterdataChannel(DR_CHANNEL_OPEN_SKY_AC_MASTERFILE, OPSKY_MD_DB_NAME) -{ - // purely informational - urlName = OPSKY_MD_CHECK_NAME; - urlLink = OPSKY_MD_CHECK_URL; - urlPopup = OPSKY_MD_CHECK_POPUP; -} - -// accept only master data requests for ICAO-type keys -bool OpenSkyAcMasterFile::AcceptRequest (const acStatUpdateTy& r) -{ - if (r.type == DATREQU_AC_MASTER && - r.acKey.eKeyType == LTFlightData::KEY_ICAO && - !ShallIgnore(r)) - { - InsertRequest(r); - return true; - } - return false; -} - -// virtual thread main function -void OpenSkyAcMasterFile::Main () -{ - // This is a communication thread's main function, set thread's name and C locale - ThreadSettings TS ("LT_OpSkyMstFile", LC_ALL_MASK); - - // Make sure we have an aircraft database file, and open it - if (!OpenDatabaseFile()) { - SHOW_MSG(logERR, "No OpenSky Aircraft Database file available!"); - SetValid(false,true); - return; - } - - // Loop to process requests - RegisterMasterDataChn(this); // Unregister myself as a master data channel - tSetRequCleared = dataRefs.GetMiscNetwTime(); - while ( shallRun() ) { - // LiveTraffic Top Level Exception Handling - try { - // if there is something to request, fetch the data and process it - if (FetchNextRequest()) { - if (LookupData() && ProcessFetchedData()) { - // reduce error count if processed successfully, as a chance to appear OK in the long run - DecErrCnt(); - } else { - // Could not find the data here, pass on the request - PassOnRequest(this, currRequ); - } - } - - // sleep for OPSKY_WAIT_NOQUEUE or if woken up for termination - // by condition variable trigger - if (!HaveAnyRequest()) - { - std::unique_lock lk(FDThreadSynchMutex); - FDThreadSynchCV.wait_for(lk,OPSKY_WAIT_NOQUEUE, - [this]{return !shallRun() || HaveAnyRequest();}); - } - - // Every 3s clear up outdated requests waiting in queue - if (CheckEverySoOften(tSetRequCleared, 3.0f)) - MaintainMasterDataRequests(); - - } catch (const std::exception& e) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); - IncErrCnt(); - } catch (...) { - LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); - IncErrCnt(); - } - } - - // Before leaving clear the request queue - std::unique_lock lock (mtxMaster); - if (bFDMainStop) // in case of all stopping just throw them away - setAcStatRequ.clear(); - else - MaintainMasterDataRequests(); // otherwise clear up and pass on - UnregisterMasterDataChn(this); // Unregister myself as a master data channel -} - -/// @brief Extract the hexId from an OpenSky ac database line, `0` if invalid -/// @details Line needs to start with a full 6-digit hex id in double quotes: "000001" -unsigned long GetHexId (const std::string& ln) -{ - // must have at least 8 characters, and double quotes at position 0 and 7: - if (ln.size() < 8 || - ln[0] != '"' || ln[7] != '"') - return 0UL; - - // Test the inside for all hex digits - if (!std::all_of(ln.begin() + 1, - ln.begin() + 7, - [](const char& ch){ return std::isxdigit(ch); })) - return 0UL; - - // convert text to number - return std::strtoul(ln.c_str()+1, nullptr, 16); -} - -/// Process looked up master data -/// @details Database file line is expected in `ln` -bool OpenSkyAcMasterFile::ProcessFetchedData () -{ - try { - // Sanity check: Less than 45 chars is impossible for a valid line (15 fields with at least 3 chars: "","","",... - if (ln.size() < ACMFF_NUM_FIELDS * 3) - return false; - - // Split the line into its fields. - // Note that we split by the 3 characters "," so that fields don't just depend on the comma - // and most but not all double quotes are already removed along the way - std::vector v = str_fields(ln, "\",\""); - if (v.size() < ACMFF_NUM_FIELDS) { - LOG_MSG(logWARN, "A/c database file line has too few fields: %s", ln.c_str()); - AddIgnore(); - return false; - } - - // The very first and very last double quotes have not yet been removed - if (!v.front().empty() && v.front().front() == '"') v.front().erase(0,1); - if (!v.back().empty() && v.back().back() == '"') v.back().erase(v.back().length()-1,1); - - // Sanity check: Hex id must match the current search request - if (std::strtoul(v[ACMFF_hexId].c_str(), nullptr, 16) != currRequ.acKey.num) { - LOG_MSG(logERR, "A/c id of fetched db line (%s) doesn't match requested id (%s)", - v[ACMFF_hexId].c_str(), currRequ.acKey.c_str()); - AddIgnore(); - return false; - } - - // Fill readable static data from the database line - LTFlightData::FDStaticData stat; - stat.reg = v[ACMFF_reg]; - stat.acTypeIcao = v[ACMFF_designator]; - stat.man = !v[ACMFF_man].empty() ? v[ACMFF_man] : - v[ACMFF_manIcao]; - stat.mdl = v[ACMFF_mdl]; - stat.catDescr = v[ACMFF_catDescr]; - stat.op = !v[ACMFF_operator].empty() ? v[ACMFF_operator] : - !v[ACMFF_owner].empty() ? v[ACMFF_owner] : - v[ACMFF_operatorCallsign]; - stat.opIcao = v[ACMFF_opIcao]; - - // Update flight data - UpdateStaticData(currRequ.acKey, stat); - return true; - } - catch (const std::runtime_error& e) { - LOG_MSG(logERR, "Couldn't process record: %s", e.what()); - IncErrCnt(); - } - return false; -} - -/// perform the file lookup -/// @details Looks up a starting position in the map of position, -/// as to seek in the database file to a near but not exact position. -/// From there, read line-by-line through the database file until we find the record we are looking for. -bool OpenSkyAcMasterFile::LookupData () -{ - ln.clear(); - try { - // Not generally good to try? Not enabled, no file? - if (!shallRun() || !fAcDb.is_open()) - return false; - - // find search starting position in the map of positions, based on on a/c identifier - mapPosTy::const_iterator iPos = mapPos.lower_bound(currRequ.acKey.num); - // iPos->first is now equal or larger to acKey, but we need equal or lower, so potentially decrement - while (iPos->first > currRequ.acKey.num) { - if (iPos == mapPos.begin()) { // cannot decrement any longer, what's wrong here...a key smaller than the first entry in the database??? - LOG_MSG(logWARN, "A/c key %06lX is smaller than first entry in database %06lX", - currRequ.acKey.num, iPos->first); - AddIgnore(); - return false; - } - --iPos; - } - - // iPos->second now points to a database file location on or before the record we seek - unsigned long lnKey = 0UL; - fAcDb.seekg(iPos->second); - do { - safeGetline(fAcDb, ln); // read one line - lnKey = GetHexId(ln); // get the a/c key from the line - } - // repeat while the line's key is smaller - while (fAcDb.good() && (!lnKey || lnKey < currRequ.acKey.num)); - - // If this is not the right record then we don't have it - if (lnKey != currRequ.acKey.num) { - AddIgnore(); - return false; - } - // else this was the key! - LOG_ASSERT(lnKey == currRequ.acKey.num); - return true; - } - catch (const std::runtime_error& e) { - LOG_MSG(logERR, "Couldn't look up record: %s", e.what()); - IncErrCnt(); - } - // technical failure: - return false; -} - -// find an aircraft database file to open/download -bool OpenSkyAcMasterFile::OpenDatabaseFile () -{ - // Get current month and year - const time_t now = time(nullptr); - struct std::tm tm = *gmtime(&now); - - // Normalize year and month to human-typical values - tm.tm_year += 1900; - tm.tm_mon++; - - // Try this month and two previous months - for (int i = 2; i >= 0; --i) { - if (TryOpenDbFile(tm.tm_year, tm.tm_mon)) - return true; - // try previous month, potentially rolling back to previous year - if (--tm.tm_mon < 1) { - --tm.tm_year; - tm.tm_mon = 12; - } - } - - // as a last resort: we _know_ that the file for DEC-2023 was there - return TryOpenDbFile(2023, 12); -} - - -// open/download the aircraft database file for the given month -/// @details After opening the file, loop over all lines -/// (~580,000) and save position information every -/// 250 lines, so that we can search faster later -/// when looking up a/c keys. -bool OpenSkyAcMasterFile::TryOpenDbFile (int year, int month) -{ - // filename of what this is about - char fileName[50] = {0}; - snprintf(fileName, sizeof(fileName), OPSKY_MD_DB_FILE, - year, month); - - try { - // Is the file available already? - std::string filePath = dataRefs.GetLTPluginPath(); - filePath += PATH_RESOURCES; - filePath += '/'; - filePath += fileName; - - // Just try to open and see what happens - fAcDb.open(filePath); - if (!fAcDb.is_open() || !fAcDb.good()) - { - fAcDb.close(); - - // file doesn't exist, try to download - std::string url = OPSKY_MD_DB_URL; - url += fileName; - LOG_MSG(logDEBUG, "Try to download %s", url.c_str()); - if (!RemoteFileDownload(url,filePath)) { - LOG_MSG(logDEBUG, "Download of %s unavailable", url.c_str()); - return false; - } - - fAcDb.open(filePath); - if (!fAcDb.is_open() || !fAcDb.good()) { // download but not OK to read? - std::remove(filePath.c_str()); // remove and bail - fAcDb.close(); - return false; - } - } - - // File is open and good to read - LOG_MSG(logDEBUG, "Processing %s as aircraft database", filePath.c_str()); - mapPos.clear(); - fAcDb.seekg(0); - - // --- Loop the file and save every 250th position --- - // hexId, reg, manIcao, man, mdl, designator, serialNum, lineNum, icaoAircraftClass, operator, operatorCallsign, opIcao, opIata, owner, catDescr - // "0000c4","N474EA","BOEING","Boeing","737-448 /SF","B734","24474","1742","L2J","","","","","","" - // "00015f","-UNKNOWN-","TAI","General Dynamics","F-16","F16","","","L1J","Baf","BELGIAN AIRFORCE","BAF","","","" - - unsigned long prevHexId = 0; - unsigned long lnNr = 0; - while (fAcDb.good()) { - const std::ifstream::pos_type pos = fAcDb.tellg(); // current position _before_ reading the line - safeGetline(fAcDb, ln); - - // Here, we are only interested in the very first field, the hexId - unsigned long hexId = GetHexId(ln); - if (hexId) { - if (hexId <= prevHexId) { - // this id not larger than last --> NOT SORTED! - LOG_MSG(logERR, "A/c database file '%s' appears not sorted at line '%s'!", - filePath.c_str(), ln.c_str()); - fAcDb.close(); - // We don't delete the file...it's invalid, but if we'd delete it we would only re-download next start again, which we want to avoid - return false; - } - prevHexId = hexId; - - // Save the position every now and then - if (lnNr % OPSKY_NUM_LN_PER_POS == 0) - mapPos.emplace(hexId, pos); - ++lnNr; - } - } - - // looks good! - fAcDb.clear(); - return true; - - } catch (const std::exception& e) { - LOG_MSG(logERR, "Could not download/open a/c database file '%s': %s", fileName, e.what()); - } catch (...) { - LOG_MSG(logERR, "Could not download/open a/c database file '%s'", fileName); - } - return false; -} +/// @file LTOpenSky.cpp +/// @brief OpenSky Network: Requests and processes live tracking and aircraft master data +/// @see https://opensky-network.org/ +/// @details Implements OpenSkyConnection and OpenSkyAcMasterdata:\n +/// - Provides a proper REST-conform URL\n +/// - Interprets the response and passes the tracking data on to LTFlightData.\n +/// @author Birger Hoppe +/// @copyright (c) 2018-2020 Birger Hoppe +/// @copyright Permission is hereby granted, free of charge, to any person obtaining a +/// copy of this software and associated documentation files (the "Software"), +/// to deal in the Software without restriction, including without limitation +/// the rights to use, copy, modify, merge, publish, distribute, sublicense, +/// and/or sell copies of the Software, and to permit persons to whom the +/// Software is furnished to do so, subject to the following conditions:\n +/// The above copyright notice and this permission notice shall be included in +/// all copies or substantial portions of the Software.\n +/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +/// THE SOFTWARE. + +// All includes are collected in one header +#include "LiveTraffic.h" + +#if IBM +#else +#include +#endif + +// +//MARK: OpenSky +// + +// Constructor +OpenSkyConnection::OpenSkyConnection () : +LTFlightDataChannel(DR_CHANNEL_OPEN_SKY_ONLINE, OPSKY_NAME) +{ + // purely informational + urlName = OPSKY_CHECK_NAME; + urlLink = OPSKY_CHECK_URL; + urlPopup = OPSKY_CHECK_POPUP; +} + +// virtual thread main function +void OpenSkyConnection::Main () +{ + // This is a communication thread's main function, set thread's name and C locale + ThreadSettings TS ("LT_OpSky", LC_ALL_MASK); + + while ( shallRun() ) { + // LiveTraffic Top Level Exception Handling + try { + // basis for determining when to be called next + tNextWakeup = std::chrono::steady_clock::now(); + + // where are we right now? + const positionTy pos (dataRefs.GetViewPos()); + + // If the camera position is valid we can request data around it + if (pos.isNormal()) { + // Next wakeup is "refresh interval" from _now_ + tNextWakeup += std::chrono::seconds(dataRefs.GetFdRefreshIntvl()); + + // fetch data and process it + if (FetchAllData(pos) && ProcessFetchedData()) + // reduce error count if processed successfully + // as a chance to appear OK in the long run + DecErrCnt(); + } + else { + // Camera position is yet invalid, retry in a second + tNextWakeup += std::chrono::seconds(1); + } + + // sleep for FD_REFRESH_INTVL or if woken up for termination + // by condition variable trigger + { + std::unique_lock lk(FDThreadSynchMutex); + FDThreadSynchCV.wait_until(lk, tNextWakeup, + [this]{return !shallRun();}); + } + + } catch (const std::exception& e) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); + IncErrCnt(); + } catch (...) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); + IncErrCnt(); + } + } +} + + +// Initialize CURL, adding OpenSky credentials +bool OpenSkyConnection::InitCurl () +{ + // Standard-init first (repeated call will just return true without effect) + if (!LTOnlineChannel::InitCurl()) + return false; + + // if there are credentials then now is the moment to add them + std::string usr, pwd; + dataRefs.GetOpenSkyCredentials(usr, pwd); + if (!usr.empty() && !pwd.empty()) { + curl_easy_setopt(pCurl, CURLOPT_USERNAME, usr.data()); + curl_easy_setopt(pCurl, CURLOPT_PASSWORD, pwd.data()); + } else { + curl_easy_setopt(pCurl, CURLOPT_USERNAME, nullptr); + curl_easy_setopt(pCurl, CURLOPT_PASSWORD, nullptr); + } + + // read headers (for remaining requests info) + curl_easy_setopt(pCurl, CURLOPT_HEADERFUNCTION, ReceiveHeader); + + return true; +} + +// read header and parse for request remaining +size_t OpenSkyConnection::ReceiveHeader(char *buffer, size_t size, size_t nitems, void *) +{ + const size_t len = nitems * size; + static size_t lenRRemain = strlen(OPSKY_RREMAIN); + static size_t lenRetry = strlen(OPSKY_RETRY); + char num[50]; + + // Remaining? + if (len > lenRRemain && + memcmp(buffer, OPSKY_RREMAIN, lenRRemain) == 0) + { + const size_t copyCnt = std::min(len-lenRRemain,sizeof(num)-1); + memcpy(num, buffer+lenRRemain, copyCnt); + num[copyCnt]=0; // zero termination + + long rRemain = std::atol(num); + // Issue a warning when coming close to the end + if (rRemain != dataRefs.OpenSkyRRemain) { + if (rRemain == 50 || rRemain == 10) { + SHOW_MSG(logWARN, "OpenSky: Only %ld requests left today for ca. %ld minutes of data", + rRemain, + (rRemain * dataRefs.GetFdRefreshIntvl()) / 60); + } + dataRefs.OpenSkyRRemain = rRemain; + if (rRemain > 0) + dataRefs.OpenSkyRetryAt.clear(); + } + } + // Retry-after-seconds? + else if (len > lenRetry && + memcmp(buffer, OPSKY_RETRY, lenRetry) == 0) + { + const size_t copyCnt = std::min(len-lenRetry,sizeof(num)-1); + memcpy(num, buffer+lenRetry, copyCnt); + num[copyCnt]=0; // zero termination + long secRetry = std::atol(num); // seconds till retry + // convert that to a local timestamp for the user to use + const std::time_t tRetry = std::time(nullptr) + secRetry; + std::strftime(num, sizeof(num), "%d-%b %H:%M", std::localtime(&tRetry)); + dataRefs.OpenSkyRetryAt = num; + dataRefs.OpenSkyRRemain = 0; + } + + // always say we processed everything, otherwise HTTP processing would stop! + return len; +} + +// put together the URL to fetch based on current view position +std::string OpenSkyConnection::GetURL (const positionTy& pos) +{ + // we add 10% to the bounding box to have some data ready once the plane is close enough for display + boundingBoxTy box (pos, double(dataRefs.GetFdStdDistance_m()) * 1.10); + char url[128] = ""; + snprintf(url, sizeof(url), + OPSKY_URL_ALL, + box.se.lat(), // lamin + box.nw.lon(), // lomin + box.nw.lat(), // lamax + box.se.lon() ); // lomax + return std::string(url); +} + +// update shared flight data structures with received flight data +// "a4d85d","UJC11 ","United States",1657226901,1657226901,-90.2035,38.8157,2758.44,false,128.1,269.54,-6.5,null,2895.6,"4102",false,0 +bool OpenSkyConnection::ProcessFetchedData () +{ + char buf[100]; + + // any a/c filter defined for debugging purposes? + std::string acFilter ( dataRefs.GetDebugAcFilter() ); + + // data is expected to be in netData string + // short-cut if there is nothing + if ( !netDataPos ) return true; + + // Only proceed in case HTTP response was OK + if (httpResponse != HTTP_OK) { + // Unauthorized? + if (httpResponse == HTTP_UNAUTHORIZED) { + SHOW_MSG(logERR, "OpenSky: Unauthorized! Verify username/password in settings."); + SetValid(false,false); + SetEnable(false); // also disable to directly allow user/pwd change...and won't work on retry anyway + return false; + } + + // Ran out of requests? + if (httpResponse == HTTP_TOO_MANY_REQU) { + SHOW_MSG(logERR, "OpenSky: Used up request credit for today, try again on %s", + dataRefs.OpenSkyRetryAt.empty() ? "" : dataRefs.OpenSkyRetryAt.c_str()); + SetValid(false,false); + return false; + } + + // Timeouts are so common recently with OpenSky that we no longer treat them as errors, + // but we inform the user every once in a while + if (httpResponse == HTTP_GATEWAY_TIMEOUT && + httpResponse == HTTP_TIMEOUT) + { + static std::chrono::time_point lastTimeoutWarn; + auto tNow = std::chrono::steady_clock::now(); + if (tNow > lastTimeoutWarn + std::chrono::minutes(5)) { + lastTimeoutWarn = tNow; + SHOW_MSG(logWARN, "%s communication unreliable due to timeouts!", pszChName); + } + } + else { // anything else is serious + IncErrCnt(); + } + return false; + } + + // now try to interpret it as JSON + JSONRootPtr pRoot (netData); + if (!pRoot) { LOG_MSG(logERR,ERR_JSON_PARSE); IncErrCnt(); return false; } + + // let's cycle the aircraft + // first get the structre's main object + JSON_Object* pObj = json_object(pRoot.get()); + if (!pObj) { LOG_MSG(logERR,ERR_JSON_MAIN_OBJECT); IncErrCnt(); return false; } + + // for determining an offset as compared to network time we need to know network time +/* Temporarily disabled, see https://forums.x-plane.org/index.php?/forums/topic/301833-aircraft-fail-to-display-buffer-times-going-up/ + double opSkyTime = jog_n(pObj, OPSKY_TIME); + if (opSkyTime > JAN_FIRST_2019) + // if reasonable add this to our time offset calculation + dataRefs.ChTsOffsetAdd(opSkyTime); +*/ + + // Cut-off time: We ignore tracking data, which is "in the past" compared to simTime + const double tsCutOff = dataRefs.GetSimTime(); + + // We need to calculate distance to current camera later on + const positionTy viewPos = dataRefs.GetViewPos(); + + // fetch the aircraft array + JSON_Array* pJAcList = json_object_get_array(pObj, OPSKY_AIRCRAFT_ARR); + if (!pJAcList) { + // a/c array not found: can just mean it is 'null' as in + // the empty result set: {"time":1541978120,"states":null} + JSON_Value* pJSONVal = json_object_get_value(pObj, OPSKY_AIRCRAFT_ARR); + if (!pJSONVal || json_type(pJSONVal) != JSONNull) { + // well...it is something else, so it is malformed, bail out + LOG_MSG(logERR,ERR_JSON_ACLIST,OPSKY_AIRCRAFT_ARR); + IncErrCnt(); + return false; + } + } + // iterate all aircraft in the received flight data (can be 0) + else for ( size_t i=0; i < json_array_get_count(pJAcList); i++ ) + { + // get the aircraft (which is just an array of values) + JSON_Array* pJAc = json_array_get_array(pJAcList,i); + if (!pJAc) { + LOG_MSG(logERR,ERR_JSON_AC,i+1,OPSKY_AIRCRAFT_ARR); + if (IncErrCnt()) + continue; + else + return false; + } + + // the key: transponder Icao code + LTFlightData::FDKeyTy fdKey (LTFlightData::KEY_ICAO, + jag_s(pJAc, OPSKY_TRANSP_ICAO)); + + // not matching a/c filter? -> skip it + if ((!acFilter.empty() && (fdKey != acFilter)) ) + { + continue; + } + + // position time + const double posTime = jag_n(pJAc, OPSKY_POS_TIME); + if (posTime <= tsCutOff) + continue; + + try { + // from here on access to fdMap guarded by a mutex + // until FD object is inserted and updated + std::unique_lock mapFdLock (mapFdMutex); + + // Check for duplicates with OGN/FLARM, potentially replaces the key type + LTFlightData::CheckDupKey(fdKey, LTFlightData::KEY_FLARM); + + // get the fd object from the map, key is the transpIcao + // this fetches an existing or, if not existing, creates a new one + LTFlightData& fd = mapFd[fdKey]; + + // also get the data access lock once and for all + // so following fetch/update calls only make quick recursive calls + std::lock_guard fdLock (fd.dataAccessMutex); + // now that we have the detail lock we can release the global one + mapFdLock.unlock(); + + // completely new? fill key fields + if ( fd.empty() ) + fd.SetKey(fdKey); + + // fill static data + LTFlightData::FDStaticData stat; + stat.country = jag_s(pJAc, OPSKY_COUNTRY); + stat.call = jag_s(pJAc, OPSKY_CALL); + while (!stat.call.empty() && stat.call.back() == ' ') // trim trailing spaces + stat.call.pop_back(); + if (!fdKey.empty()) { + snprintf(buf, sizeof(buf), OPSKY_SLUG_FMT, fdKey.num); + stat.slug = buf; + } + + // dynamic data + { // unconditional...block is only for limiting local variables + LTFlightData::FDDynamicData dyn; + + // non-positional dynamic data + dyn.radar.code = (long)jag_sn(pJAc, OPSKY_RADAR_CODE); + dyn.gnd = jag_b(pJAc, OPSKY_GND); + dyn.heading = jag_n_nan(pJAc, OPSKY_HEADING); + dyn.spd = jag_n(pJAc, OPSKY_SPD); + dyn.vsi = jag_n(pJAc, OPSKY_VSI); + dyn.ts = posTime; + dyn.pChannel = this; + + // position + const double baroAlt_m = jag_n_nan(pJAc, OPSKY_BARO_ALT); + const double geoAlt_m = BaroAltToGeoAlt_m(baroAlt_m, dataRefs.GetPressureHPA()); + positionTy pos (jag_n_nan(pJAc, OPSKY_LAT), + jag_n_nan(pJAc, OPSKY_LON), + geoAlt_m, + posTime, + dyn.heading); + pos.f.onGrnd = dyn.gnd ? GND_ON : GND_OFF; + + // Update static data + fd.UpdateData(std::move(stat), pos.dist(viewPos)); + + // position is rather important, we check for validity + // (we do allow alt=NAN if on ground as this is what OpenSky returns) + if ( pos.isNormal(true) ) + fd.AddDynData(dyn, 0, 0, &pos); + else + LOG_MSG(logDEBUG,ERR_POS_UNNORMAL,fdKey.c_str(),pos.dbgTxt().c_str()); + } + } catch(const std::system_error& e) { + LOG_MSG(logERR, ERR_LOCK_ERROR, "mapFd", e.what()); + } + } + + // success + return true; +} + + +// get status info, including remaining requests +std::string OpenSkyConnection::GetStatusText () const +{ + std::string s = LTChannel::GetStatusText(); + if (IsValid()) { + if (dataRefs.OpenSkyRRemain < LONG_MAX) + { + s += " | "; + s += std::to_string(dataRefs.OpenSkyRRemain); + s += " requests left today"; + } + } else { + if (!dataRefs.OpenSkyRetryAt.empty()) + { + s += ", retry at "; + s += dataRefs.OpenSkyRetryAt; + } + } + + return s; +} + + + +// +//MARK: OpenSkyAcMasterdata +// + +// Constructor +OpenSkyAcMasterdata::OpenSkyAcMasterdata () : +LTACMasterdataChannel(DR_CHANNEL_OPEN_SKY_AC_MASTERDATA, OPSKY_MD_NAME) +{ + // purely informational + urlName = OPSKY_MD_CHECK_NAME; + urlLink = OPSKY_MD_CHECK_URL; + urlPopup = OPSKY_MD_CHECK_POPUP; +} + +// accept requests that aren't in the ignore lists +bool OpenSkyAcMasterdata::AcceptRequest (const acStatUpdateTy& r) +{ + if ((r.type == DATREQU_ROUTE || // accepting all kinds of call signs + r.acKey.eKeyType == LTFlightData::KEY_ICAO) && // but only ICAO-typed master data requests + !ShallIgnore(r)) + { + InsertRequest(r); + return true; + } + return false; +} + +// virtual thread main function +void OpenSkyAcMasterdata::Main () +{ + // This is a communication thread's main function, set thread's name and C locale + ThreadSettings TS ("LT_OpSkyMaster", LC_ALL_MASK); + RegisterMasterDataChn(this); // Register myself as a master data channel + tSetRequCleared = dataRefs.GetMiscNetwTime(); + + while ( shallRun() ) { + // LiveTraffic Top Level Exception Handling + try { + // if there is something to request, fetch the data and process it + if (FetchNextRequest()) + { + if (FetchAllData(positionTy()) && ProcessFetchedData()) { + // reduce error count if processed successfully, as a chance to appear OK in the long run + DecErrCnt(); + } else { + // Could not find the data here, pass on the request + PassOnRequest(this, currRequ); + } + } + + // We must wait a moment between any two requests just not to overload the server + std::this_thread::sleep_for(OPSKY_WAIT_BETWEEN); + + // sleep a bit or until woken up for termination by condition variable trigger + if (!HaveAnyRequest()) + { + std::unique_lock lk(FDThreadSynchMutex); + FDThreadSynchCV.wait_for(lk, OPSKY_WAIT_NOQUEUE, + [this]{return !shallRun() || HaveAnyRequest();}); + } + + // Every 3s clear up outdated requests waiting in queue + if (CheckEverySoOften(tSetRequCleared, 3.0f)) + MaintainMasterDataRequests(); + + } catch (const std::exception& e) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); + IncErrCnt(); + } catch (...) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); + IncErrCnt(); + } + } + + // Before leaving clear the request queue + std::unique_lock lock (mtxMaster); + if (bFDMainStop) // in case of all stopping just throw them away + setAcStatRequ.clear(); + else + MaintainMasterDataRequests(); // otherwise clear up and pass on + UnregisterMasterDataChn(this); // Unregister myself as a master data channel +} + + +// Returns the master data or route URL to query +std::string OpenSkyAcMasterdata::GetURL (const positionTy& /*pos*/) +{ + switch (currRequ.type) { + case DATREQU_AC_MASTER: + return std::string(OPSKY_MD_URL) + URLEncode(currRequ.acKey.key); + case DATREQU_ROUTE: + return std::string(OPSKY_ROUTE_URL) + URLEncode(currRequ.callSign); + case DATREQU_NONE: + return std::string(); + } + return std::string(); +} + +// process each master data line read from OpenSky +bool OpenSkyAcMasterdata::ProcessFetchedData () +{ + // If the requested data just wasn't found add it to the ignore list + if (httpResponse == HTTP_NOT_FOUND) { + AddIgnore(); + return false; + } + // Any other result or no message is technically not OK + else if (httpResponse != HTTP_OK || !netDataPos) { + IncErrCnt(); + return false; + } + + // Try to interpret is as JSON and get the main JSON object + JSONRootPtr pRoot (netData); + if (!pRoot) { LOG_MSG(logERR,ERR_JSON_PARSE); IncErrCnt(); return false; } + JSON_Object* pObj = json_object(pRoot.get()); + if (!pObj) { LOG_MSG(logERR,ERR_JSON_MAIN_OBJECT); IncErrCnt(); return false; } + + // Pass on the further processing depending on the request type + switch (currRequ.type) { + case DATREQU_AC_MASTER: + return ProcessMasterData(pObj); + case DATREQU_ROUTE: + return ProcessRouteInfo(pObj); + case DATREQU_NONE: + break; + } + return false; +} + + +// Process received aircraft master data +bool OpenSkyAcMasterdata::ProcessMasterData (JSON_Object* pJAc) +{ + LTFlightData::FDKeyTy fdKey; // the key: transponder Icao code, filled from response! + LTFlightData::FDStaticData statDat; // here we collect the master data + + // fetch values from the online data + fdKey.SetKey(LTFlightData::KEY_ICAO, + jog_s(pJAc, OPSKY_MD_TRANSP_ICAO)); + statDat.reg = jog_s(pJAc, OPSKY_MD_REG); + statDat.country = jog_s(pJAc, OPSKY_MD_COUNTRY); + statDat.acTypeIcao = jog_s(pJAc, OPSKY_MD_AC_TYPE_ICAO); + statDat.man = jog_s(pJAc, OPSKY_MD_MAN); + statDat.mdl = jog_s(pJAc, OPSKY_MD_MDL); + statDat.catDescr = jog_s(pJAc, OPSKY_MD_CAT_DESCR); + statDat.op = jog_s(pJAc, OPSKY_MD_OP); + statDat.opIcao = jog_s(pJAc, OPSKY_MD_OP_ICAO); + + // -- Ground vehicle identification -- + // OpenSky only delivers "category description" and has a + // pretty clear indicator for a ground vehicle + if (statDat.acTypeIcao.empty() && // don't know a/c type yet + (statDat.catDescr.find(OPSKY_MD_TEXT_VEHICLE) != std::string::npos || + // I'm having the feeling that if nearly all is empty and the category description is "No Info" then it's often also a ground vehicle + (statDat.catDescr.find(OPSKY_MD_TEXT_NO_CAT) != std::string::npos && + statDat.man.empty() && + statDat.mdl.empty() && + statDat.opIcao.empty()))) + { + // we assume ground vehicle + statDat.acTypeIcao = dataRefs.GetDefaultCarIcaoType(); + // The category description usually is something like + // "Surface Vehicle – Service Vehicle" + // Save the latter part if we have no model info yet + if (statDat.mdl.empty() && + statDat.catDescr.find(OPSKY_MD_TEXT_VEHICLE) != std::string::npos && + statDat.catDescr.length() > OPSKY_MD_TEXT_VEHICLE_LEN) + { + statDat.mdl = statDat.catDescr.c_str() + OPSKY_MD_TEXT_VEHICLE_LEN; + } + } + // Replace type GRND with our default car type, too + else if (statDat.acTypeIcao == "GRND" || statDat.acTypeIcao == "GND") + statDat.acTypeIcao = dataRefs.GetDefaultCarIcaoType(); + + // Perform the update + UpdateStaticData(fdKey, statDat); + return true; +} + + +// Process received route info +bool OpenSkyAcMasterdata::ProcessRouteInfo (JSON_Object* pJRoute) +{ + LTFlightData::FDStaticData statDat; // here we collect the master data + + // fetch values from the online data + // route is an array of typically 2 entries, but can have more + // "route":["EDDM","LIMC"] + JSON_Array* pJRArr = json_object_get_array(pJRoute, OPSKY_ROUTE_ROUTE); + if (pJRArr) { + size_t cnt = json_array_get_count(pJRArr); + for (size_t i = 0; i < cnt; i++) + statDat.stops.push_back(jag_s(pJRArr, i)); + } + + // flight number: made up of IATA and actual number + statDat.flight = jog_s(pJRoute,OPSKY_ROUTE_OP_IATA); + double flightNr = jog_n_nan(pJRoute,OPSKY_ROUTE_FLIGHT_NR); + if (!std::isnan(flightNr)) + statDat.flight += std::to_string(lround(flightNr)); + + // update the a/c's master data + UpdateStaticData(currRequ.acKey, statDat); + return true; +} + +// +// MARK: OpenSky Master Data File +// + + +// Constructor +OpenSkyAcMasterFile::OpenSkyAcMasterFile () : +LTACMasterdataChannel(DR_CHANNEL_OPEN_SKY_AC_MASTERFILE, OPSKY_MD_DB_NAME) +{ + // purely informational + urlName = OPSKY_MD_CHECK_NAME; + urlLink = OPSKY_MD_CHECK_URL; + urlPopup = OPSKY_MD_CHECK_POPUP; +} + +// accept only master data requests for ICAO-type keys +bool OpenSkyAcMasterFile::AcceptRequest (const acStatUpdateTy& r) +{ + if (r.type == DATREQU_AC_MASTER && + r.acKey.eKeyType == LTFlightData::KEY_ICAO && + !ShallIgnore(r)) + { + InsertRequest(r); + return true; + } + return false; +} + +// virtual thread main function +void OpenSkyAcMasterFile::Main () +{ + // This is a communication thread's main function, set thread's name and C locale + ThreadSettings TS ("LT_OpSkyMstFile", LC_ALL_MASK); + + // Make sure we have an aircraft database file, and open it + if (!OpenDatabaseFile()) { + SHOW_MSG(logERR, "No OpenSky Aircraft Database file available!"); + SetValid(false,true); + return; + } + + // Loop to process requests + RegisterMasterDataChn(this); // Unregister myself as a master data channel + tSetRequCleared = dataRefs.GetMiscNetwTime(); + while ( shallRun() ) { + // LiveTraffic Top Level Exception Handling + try { + // if there is something to request, fetch the data and process it + if (FetchNextRequest()) { + if (LookupData() && ProcessFetchedData()) { + // reduce error count if processed successfully, as a chance to appear OK in the long run + DecErrCnt(); + } else { + // Could not find the data here, pass on the request + PassOnRequest(this, currRequ); + } + } + + // sleep for OPSKY_WAIT_NOQUEUE or if woken up for termination + // by condition variable trigger + if (!HaveAnyRequest()) + { + std::unique_lock lk(FDThreadSynchMutex); + FDThreadSynchCV.wait_for(lk,OPSKY_WAIT_NOQUEUE, + [this]{return !shallRun() || HaveAnyRequest();}); + } + + // Every 3s clear up outdated requests waiting in queue + if (CheckEverySoOften(tSetRequCleared, 3.0f)) + MaintainMasterDataRequests(); + + } catch (const std::exception& e) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, e.what()); + IncErrCnt(); + } catch (...) { + LOG_MSG(logERR, ERR_TOP_LEVEL_EXCEPTION, "(unknown type)"); + IncErrCnt(); + } + } + + // Before leaving clear the request queue + std::unique_lock lock (mtxMaster); + if (bFDMainStop) // in case of all stopping just throw them away + setAcStatRequ.clear(); + else + MaintainMasterDataRequests(); // otherwise clear up and pass on + UnregisterMasterDataChn(this); // Unregister myself as a master data channel +} + +/// @brief Extract the hexId from an OpenSky ac database line, `0` if invalid +/// @details Line needs to start with a full 6-digit hex id in double quotes: "000001" +unsigned long GetHexId (const std::string& ln) +{ + // must have at least 8 characters, and double quotes at position 0 and 7: + if (ln.size() < 8 || + ln[0] != '"' || ln[7] != '"') + return 0UL; + + // Test the inside for all hex digits + if (!std::all_of(ln.begin() + 1, + ln.begin() + 7, + [](const char& ch){ return std::isxdigit(ch); })) + return 0UL; + + // convert text to number + return std::strtoul(ln.c_str()+1, nullptr, 16); +} + +/// Process looked up master data +/// @details Database file line is expected in `ln` +bool OpenSkyAcMasterFile::ProcessFetchedData () +{ + try { + // Sanity check: Less than 45 chars is impossible for a valid line (15 fields with at least 3 chars: "","","",... + if (ln.size() < ACMFF_NUM_FIELDS * 3) + return false; + + // Split the line into its fields. + // Note that we split by the 3 characters "," so that fields don't just depend on the comma + // and most but not all double quotes are already removed along the way + std::vector v = str_fields(ln, "\",\""); + if (v.size() < ACMFF_NUM_FIELDS) { + LOG_MSG(logWARN, "A/c database file line has too few fields: %s", ln.c_str()); + AddIgnore(); + return false; + } + + // The very first and very last double quotes have not yet been removed + if (!v.front().empty() && v.front().front() == '"') v.front().erase(0,1); + if (!v.back().empty() && v.back().back() == '"') v.back().erase(v.back().length()-1,1); + + // Sanity check: Hex id must match the current search request + if (std::strtoul(v[ACMFF_hexId].c_str(), nullptr, 16) != currRequ.acKey.num) { + LOG_MSG(logERR, "A/c id of fetched db line (%s) doesn't match requested id (%s)", + v[ACMFF_hexId].c_str(), currRequ.acKey.c_str()); + AddIgnore(); + return false; + } + + // Fill readable static data from the database line + LTFlightData::FDStaticData stat; + stat.reg = v[ACMFF_reg]; + stat.acTypeIcao = v[ACMFF_designator]; + stat.man = !v[ACMFF_man].empty() ? v[ACMFF_man] : + v[ACMFF_manIcao]; + stat.mdl = v[ACMFF_mdl]; + stat.catDescr = v[ACMFF_catDescr]; + stat.op = !v[ACMFF_operator].empty() ? v[ACMFF_operator] : + !v[ACMFF_owner].empty() ? v[ACMFF_owner] : + v[ACMFF_operatorCallsign]; + stat.opIcao = v[ACMFF_opIcao]; + + // Update flight data + UpdateStaticData(currRequ.acKey, stat); + return true; + } + catch (const std::runtime_error& e) { + LOG_MSG(logERR, "Couldn't process record: %s", e.what()); + IncErrCnt(); + } + return false; +} + +/// perform the file lookup +/// @details Looks up a starting position in the map of position, +/// as to seek in the database file to a near but not exact position. +/// From there, read line-by-line through the database file until we find the record we are looking for. +bool OpenSkyAcMasterFile::LookupData () +{ + ln.clear(); + try { + // Not generally good to try? Not enabled, no file? + if (!shallRun() || !fAcDb.is_open()) + return false; + + // find search starting position in the map of positions, based on on a/c identifier + mapPosTy::const_iterator iPos = mapPos.lower_bound(currRequ.acKey.num); + // iPos->first is now equal or larger to acKey, but we need equal or lower, so potentially decrement + while (iPos->first > currRequ.acKey.num) { + if (iPos == mapPos.begin()) { // cannot decrement any longer, what's wrong here...a key smaller than the first entry in the database??? + LOG_MSG(logWARN, "A/c key %06lX is smaller than first entry in database %06lX", + currRequ.acKey.num, iPos->first); + AddIgnore(); + return false; + } + --iPos; + } + + // iPos->second now points to a database file location on or before the record we seek + unsigned long lnKey = 0UL; + fAcDb.seekg(iPos->second); + do { + safeGetline(fAcDb, ln); // read one line + lnKey = GetHexId(ln); // get the a/c key from the line + } + // repeat while the line's key is smaller + while (fAcDb.good() && (!lnKey || lnKey < currRequ.acKey.num)); + + // If this is not the right record then we don't have it + if (lnKey != currRequ.acKey.num) { + AddIgnore(); + return false; + } + // else this was the key! + LOG_ASSERT(lnKey == currRequ.acKey.num); + return true; + } + catch (const std::runtime_error& e) { + LOG_MSG(logERR, "Couldn't look up record: %s", e.what()); + IncErrCnt(); + } + // technical failure: + return false; +} + +// find an aircraft database file to open/download +bool OpenSkyAcMasterFile::OpenDatabaseFile () +{ + // Get current month and year + const time_t now = time(nullptr); + struct std::tm tm = *gmtime(&now); + + // Normalize year and month to human-typical values + tm.tm_year += 1900; + tm.tm_mon++; + + // Try this month and two previous months + for (int i = 2; i >= 0; --i) { + if (TryOpenDbFile(tm.tm_year, tm.tm_mon)) + return true; + // try previous month, potentially rolling back to previous year + if (--tm.tm_mon < 1) { + --tm.tm_year; + tm.tm_mon = 12; + } + } + + // as a last resort: we _know_ that the file for DEC-2023 was there + return TryOpenDbFile(2023, 12); +} + + +// open/download the aircraft database file for the given month +/// @details After opening the file, loop over all lines +/// (~580,000) and save position information every +/// 250 lines, so that we can search faster later +/// when looking up a/c keys. +bool OpenSkyAcMasterFile::TryOpenDbFile (int year, int month) +{ + // filename of what this is about + char fileName[50] = {0}; + snprintf(fileName, sizeof(fileName), OPSKY_MD_DB_FILE, + year, month); + + try { + // Is the file available already? + const std::string fileDir = dataRefs.GetLTPluginPath() + PATH_RESOURCES + '/'; + const std::string filePath = fileDir + fileName; + + // Just try to open and see what happens + fAcDb.open(filePath); + if (!fAcDb.is_open() || !fAcDb.good()) + { + fAcDb.close(); + + // file doesn't exist, try to download + std::string url = OPSKY_MD_DB_URL; + url += fileName; + LOG_MSG(logDEBUG, "Try to download %s", url.c_str()); + if (!RemoteFileDownload(url,filePath)) { + LOG_MSG(logDEBUG, "Download of %s unavailable", url.c_str()); + return false; + } + + fAcDb.open(filePath); + if (!fAcDb.is_open() || !fAcDb.good()) { // download but not OK to read? + std::remove(filePath.c_str()); // remove and bail + fAcDb.close(); + return false; + } + } + + // File is open and good to read + LOG_MSG(logDEBUG, "Processing %s as aircraft database", filePath.c_str()); + mapPos.clear(); + fAcDb.seekg(0); + + // --- Loop the file and save every 250th position --- + // hexId, reg, manIcao, man, mdl, designator, serialNum, lineNum, icaoAircraftClass, operator, operatorCallsign, opIcao, opIata, owner, catDescr + // "0000c4","N474EA","BOEING","Boeing","737-448 /SF","B734","24474","1742","L2J","","","","","","" + // "00015f","-UNKNOWN-","TAI","General Dynamics","F-16","F16","","","L1J","Baf","BELGIAN AIRFORCE","BAF","","","" + + unsigned long prevHexId = 0; + unsigned long lnNr = 0; + while (fAcDb.good()) { + const std::ifstream::pos_type pos = fAcDb.tellg(); // current position _before_ reading the line + safeGetline(fAcDb, ln); + + // Here, we are only interested in the very first field, the hexId + unsigned long hexId = GetHexId(ln); + if (hexId) { + if (hexId <= prevHexId) { + // this id not larger than last --> NOT SORTED! + LOG_MSG(logERR, "A/c database file '%s' appears not sorted at line '%s'!", + filePath.c_str(), ln.c_str()); + fAcDb.close(); + // We don't delete the file...it's invalid, but if we'd delete it we would only re-download next start again, which we want to avoid + return false; + } + prevHexId = hexId; + + // Save the position every now and then + if (lnNr % OPSKY_NUM_LN_PER_POS == 0) + mapPos.emplace(hexId, pos); + ++lnNr; + } + } + + // looks good! + fAcDb.clear(); + + // Lastly, we remove all _other_ database files given that each takes up 50MB of disk space + // (Can't use XPLMGetDirectoryContents in non-main thread, + // using std::filesystem crashed CURL... + // so we go back to basic POSIX C and native Windows) + { + std::vector vToBeDeleted; +#if IBM + WIN32_FIND_DATA data = { 0 }; + // Search already only for files that _look_ like database files + HANDLE h = FindFirstFileA((fileDir + OPSKY_MD_DB_FILE_BEGIN + '*').c_str(), &data); + if (h != INVALID_HANDLE_VALUE) { + do { + if (!striequal(data.cFileName, fileName)) // Skip the actual file that we just processed + vToBeDeleted.emplace_back(data.cFileName); + } while (FindNextFileA(h, &data)); + FindClose(h); + } +#else + // https://stackoverflow.com/a/4204758 + DIR *d = nullptr; + struct dirent *dir = nullptr; + d = opendir(fileDir.c_str()); + if (d) { + while ((dir = readdir(d)) != NULL) { + // If begins like a database file but is not the one we just processed + std::string f = dir->d_name; + if (stribeginwith(f, OPSKY_MD_DB_FILE_BEGIN) && + !striequal(f, fileName)) + vToBeDeleted.emplace_back(std::move(f)); + } + closedir(d); + } +#endif + // Now delete what we remembered + for (const std::string& p: vToBeDeleted) + std::remove((fileDir+p).c_str()); + } + + return true; + + } catch (const std::exception& e) { + LOG_MSG(logERR, "Could not download/open a/c database file '%s': %s", fileName, e.what()); + } catch (...) { + LOG_MSG(logERR, "Could not download/open a/c database file '%s'", fileName); + } + return false; +} diff --git a/Src/LTRealTraffic.cpp b/Src/LTRealTraffic.cpp index 1b85fc8..c40538b 100644 --- a/Src/LTRealTraffic.cpp +++ b/Src/LTRealTraffic.cpp @@ -38,13 +38,15 @@ // MARK: RealTraffic Connection // -// fill from `current` data -RealTrafficConnection::WxTy& RealTrafficConnection::WxTy::operator=(const CurrTy& o) +// Set all relevant values +void RealTrafficConnection::WxTy::set(double qnh, const CurrTy& o, bool bResetErr) { + QNH = qnh; pos = o.pos; tOff = o.tOff; - time = std::chrono::steady_clock::now(); - return *this; + next = std::chrono::steady_clock::now() + RT_DRCT_WX_WAIT; + if (bResetErr) + nErr = 0; } // Constructor doesn't do much @@ -90,7 +92,10 @@ std::string RealTrafficConnection::GetStatusText () const // --- Direct Connection? --- if (eConnType == RT_CONN_REQU_REPL) { - std::string s = LTChannel::GetStatusText(); + std::string s = + curr.eRequType == CurrTy::RT_REQU_AUTH ? "Authenticating..." : + curr.eRequType == CurrTy::RT_REQU_WEATHER ? "Fetching weather..." : + LTChannel::GetStatusText(); if (tsAdjust > 1.0) { // historic data? snprintf(sIntvl, sizeof(sIntvl), MSG_RT_ADJUST, GetAdjustTSText().c_str()); @@ -258,12 +263,15 @@ void RealTrafficConnection::SetRequType (const positionTy& _pos) if (curr.sGUID.empty()) // have no GUID? Need authentication curr.eRequType = CurrTy::RT_REQU_AUTH; - else if ((rtWx.nErr < RT_DRCT_MAX_WX_ERR) && // not seen too many errors yet - (std::isnan(rtWx.QNH) || // no Weather, or wrong time offset, or outdated, or moved too far away? + else if ((std::isnan(rtWx.QNH) || // no Weather, or wrong time offset, or outdated, or moved too far away? std::labs(curr.tOff - rtWx.tOff) > 120 || - std::chrono::steady_clock::now() - rtWx.time > RT_DRCT_WX_WAIT || + std::chrono::steady_clock::now() >= rtWx.next || rtWx.pos.distRoughSqr(curr.pos) > (RT_DRCT_WX_DIST*RT_DRCT_WX_DIST))) + { curr.eRequType = CurrTy::RT_REQU_WEATHER; + if (std::labs(curr.tOff - rtWx.tOff) > 120) // if changing the timeoffset (request other historic data) then we must have new weather before proceeding + rtWx.QNH = NAN; + } else // in all other cases we ask for traffic data curr.eRequType = CurrTy::RT_REQU_TRAFFIC; @@ -437,19 +445,46 @@ bool RealTrafficConnection::ProcessFetchedData () if (curr.eRequType == CurrTy::RT_REQU_WEATHER) { // We are interested in just a single value: local Pressure const double wxSLP = jog_n_nan(pObj, "data.locWX.SLP"); - if (std::isnan(wxSLP) || wxSLP < 800.0) { - LOG_MSG(logERR, "RealTraffic returned no or invalid local pressure %.1f:\n%s", - wxSLP, netData); - rrlWait = std::chrono::seconds(2); // isn't exactly clear how quickly we can repeat weather requests! - if (++rtWx.nErr >= RT_DRCT_MAX_WX_ERR) { // Too many WX errors? - SHOW_MSG(logERR, "Too many errors trying to fetch RealTraffic weather, will continue without; planes may appear at slightly wrong altitude."); + // Error in locWX data? + std::string s = jog_s(pObj, "data.locWX.Error"); // sometimes errors are given in a specific field + if (s.empty() && + !strcmp(jog_s(pObj, "data.locWX.Info"), "TinyDelta")) // if we request too often then Info is 'TinyDelta' + s = "TinyDelta"; + // Any error, either explicitely or because local pressure is bogus? + if (!s.empty() || std::isnan(wxSLP) || wxSLP < 800.0) + { + if (s == "File requested") { + // Error "File requested" often occurs when requesting historic weather that isn't cached on the server, so we only issue debug-level message + LOG_MSG(logDEBUG, "Weather details being fetched at RealTraffic, will try again in 60s"); + } else { + // Anything else is unexpected + if (!s.empty()) { + LOG_MSG(logERR, "Requesting RealTraffic weather returned error '%s':\n%s", + s.c_str(), netData); + } else { + LOG_MSG(logERR, "RealTraffic returned no or invalid local pressure %.1f:\n%s", + wxSLP, netData); + } + } + // one more error + ++rtWx.nErr; + // If we don't yet have any pressure... + if (std::isnan(rtWx.QNH)) { + // Too many WX errors? We give up and just use standard pressure + if (rtWx.nErr >= RT_DRCT_MAX_WX_ERR) { + SHOW_MSG(logERR, "Too many errors trying to fetch RealTraffic weather, will continue without; planes may appear at slightly wrong altitude."); + rtWx.set(HPA_STANDARD, curr, false); + } else { + // We will request weather directly again, but need to wait 60s for it + rrlWait = std::chrono::seconds(60); + } } return false; } + + // Successfully received weather information LOG_MSG(logDEBUG, "Received RealTraffic locWX.SLP = %.1f", wxSLP); - rtWx.QNH = wxSLP; // Save new QNH - rtWx = curr; // and how it was requested - rtWx.nErr = 0; // reset the error counter as we now received good data + rtWx.set(wxSLP, curr); // Save new QNH return true; } diff --git a/Src/LiveTraffic.cpp b/Src/LiveTraffic.cpp index ef8f537..19a197f 100755 --- a/Src/LiveTraffic.cpp +++ b/Src/LiveTraffic.cpp @@ -46,6 +46,7 @@ enum menuItems { MENU_ID_TOGGLE_AIRCRAFT, MENU_ID_HAVE_TCAS, MENU_ID_TOGGLE_LABELS, + MENU_ID_TOGGLE_AC_AHEAD, MENU_ID_SETTINGS_UI, MENU_ID_HELP, MENU_ID_HELP_DOCUMENTATION, @@ -55,6 +56,8 @@ enum menuItems { MENU_ID_HELP_AC_INFO_WND, MENU_ID_HELP_SETTINGS, MENU_ID_HELP_INSTALL_CSL, + MENU_ID_HELP_SUPPORT_FORUM, + MENU_ID_HELP_SUPPORT_HOWTO, MENU_ID_NEWVER, #ifdef DEBUG MENU_ID_RELOAD_PLUGINS, @@ -102,6 +105,25 @@ void MenuHandler(void * /*mRef*/, void * iRef) XPLMCheckMenuItem(menuID, aMenuItems[MENU_ID_TOGGLE_LABELS], dataRefs.ToggleLabelDraw() ? xplm_Menu_Checked : xplm_Menu_Unchecked); break; + case MENU_ID_TOGGLE_AC_AHEAD: + { + bool bMenuActive = false; + const LTFlightData* pfdFocus = LTFlightData::FindFocusAc(dataRefs.GetViewHeading()); + if (pfdFocus && pfdFocus->hasAc()) { + LTAircraft* pAc = pfdFocus->GetAircraft(); + if (pAc->IsVisible()) // if visible + pAc->SetVisible(false); // hide + else { // else + pAc->SetVisible(true); // show + pAc->SetAutoVisible(true); // and (re)activate auto-show + } + bMenuActive = !pAc->IsVisible(); + } + XPLMCheckMenuItem(menuID, aMenuItems[MENU_ID_TOGGLE_AC_AHEAD], + bMenuActive ? xplm_Menu_Checked : xplm_Menu_Unchecked); + break; + } + case MENU_ID_SETTINGS_UI: XPLMCheckMenuItem(menuID,aMenuItems[MENU_ID_SETTINGS_UI], LTSettingsUI::ToggleDisplay() ? xplm_Menu_Checked : xplm_Menu_Unchecked); @@ -130,7 +152,11 @@ void MenuHandler(void * /*mRef*/, void * iRef) void MenuHandlerHelp (void * /*mRef*/, void * iRef) { const char* helpPath = static_cast(iRef); - LTOpenHelp(helpPath); + // if it starts with https it is a full URL, otherwise a help sub path + if (stribeginwith(helpPath, "https://")) + LTOpenURL(helpPath); + else + LTOpenHelp(helpPath); } // the "Aircraft displayed" menu item includes the number of displayed a/c @@ -186,6 +212,11 @@ void MenuUpdateAllItemStatus() dataRefs.AwaitingAIControl() ? MENU_HAVE_TCAS_REQUSTD : MENU_HAVE_TCAS, 0); + // Is the aircraft ahead hidden or visible? + const LTFlightData* pfdFocus = LTFlightData::FindFocusAc(dataRefs.GetViewHeading()); + XPLMCheckMenuItem(menuID, aMenuItems[MENU_ID_TOGGLE_AC_AHEAD], + pfdFocus && pfdFocus->hasAc() && !pfdFocus->GetAircraft()->IsVisible() ? xplm_Menu_Checked : xplm_Menu_Unchecked); + // Is Settings window open? XPLMCheckMenuItem(menuID,aMenuItems[MENU_ID_SETTINGS_UI], LTSettingsUI::IsDisplayed() ? xplm_Menu_Checked : xplm_Menu_Unchecked); @@ -299,6 +330,12 @@ bool RegisterMenuItem () dataRefs.cmdLT[CR_LABELS_TOGGLE]); XPLMCheckMenuItem(menuID,aMenuItems[MENU_ID_TOGGLE_LABELS], dataRefs.ShallDrawLabels() ? xplm_Menu_Checked : xplm_Menu_Unchecked); + + // Toggle visibility of aircraft ahead + aMenuItems[MENU_ID_TOGGLE_AC_AHEAD] = + AppendMenuItem(menuID, MENU_TOGGLE_AC_AHEAD, (void *)MENU_ID_TOGGLE_AC_AHEAD, + dataRefs.cmdLT[CR_TOGGLE_AC_AHEAD]); + // Separator XPLMAppendMenuSeparator(menuID); @@ -323,6 +360,9 @@ bool RegisterMenuItem () aMenuItems[MENU_ID_HELP_AC_INFO_WND] = XPLMAppendMenuItem(menuHelpID, MENU_HELP_AC_INFO_WND, (void*)HELP_AC_INFO_WND,1); aMenuItems[MENU_ID_HELP_SETTINGS] = XPLMAppendMenuItem(menuHelpID, MENU_HELP_SETTINGS, (void*)HELP_SETTINGS,1); aMenuItems[MENU_ID_HELP_INSTALL_CSL] = XPLMAppendMenuItem(menuHelpID, MENU_HELP_INSTALL_CSL, (void*)HELP_INSTALL_CSL,1); + XPLMAppendMenuSeparator(menuHelpID); + aMenuItems[MENU_ID_HELP_SUPPORT_FORUM]= XPLMAppendMenuItem(menuHelpID, MENU_HELP_SUPPORT_FORUM, (void*)URL_SUPPORT_FORUM,1); + aMenuItems[MENU_ID_HELP_SUPPORT_HOWTO]= XPLMAppendMenuItem(menuHelpID, MENU_HELP_SUPPORT_HOWTO, (void*)URL_SUPPORT_HOWTO,1); #ifdef DEBUG // Separator @@ -370,6 +410,7 @@ struct cmdMenuMap { { CR_AC_DISPLAYED, MENU_ID_TOGGLE_AIRCRAFT }, { CR_AC_TCAS_CONTROLLED, MENU_ID_HAVE_TCAS }, { CR_LABELS_TOGGLE, MENU_ID_TOGGLE_LABELS }, + { CR_TOGGLE_AC_AHEAD, MENU_ID_TOGGLE_AC_AHEAD }, { CR_SETTINGS_UI, MENU_ID_SETTINGS_UI }, }; diff --git a/Src/SettingsUI.cpp b/Src/SettingsUI.cpp index 16df3f5..8d36fd2 100644 --- a/Src/SettingsUI.cpp +++ b/Src/SettingsUI.cpp @@ -585,8 +585,10 @@ void LTSettingsUI::buildInterface() std::strftime(s, sizeof(s), "%d-%b-%Y %H:%M", &tmOfs); ImGui::TextUnformatted(s); ImGui::SameLine(); - if (ImGui::Button("Modify")) + if (ImGui::Button("Modify")) { bRTModifyTOfs = true; + tmRTManTOfs.tm_mday = 0; // make sure the timestamp-to-be-edited is refreshed according to relative time passed + } } else { // modifying the manual time offset if (!tmRTManTOfs.tm_mday) // first time edit? tmRTManTOfs = tmOfs; diff --git a/docs/readme.html b/docs/readme.html index 7c93a7c..2527456 100755 --- a/docs/readme.html +++ b/docs/readme.html @@ -129,16 +129,56 @@

Support

Release Notes

-

The issue number links refer to issue on GitHub with (often technical) details.

+

The issue number links refer to issues on GitHub with (often technical) details.

v3

-

v3.5.0

+

v3.5.1

Update: In case of doubt you can always just copy all files from the archive over the files of your existing installation.

+

At least copy the following files, which have changed compared to v3.5.0:

+
    +
  • lin|mac|win_x64/LiveTraffic.xpl
    • +
+ +

Change log:

+ +
    +
  • Hotfix to remove OpenSky Network from network time synchronization, to prevent + unreasonable buffering time from happening. + OpenSky Network also causes other network errors and will likely disable itself after some attempts + anyway. + See here for discussion and options. +
    • +
  • Not designated a Beta version as v3.5.0 accidently was. No usage time limit.
    + You want to set Settings > Advanced > Logging > Log.txt logging level + back to "Warning" to reduce output to Log.txt. + That setting got set to "Debug" while running a Beta version. +
    • +
  • RealTraffic with historic data: +
      +
    • Fixed fetching historic weather, more robust now.
      • +
    • Fixed glitch in Settings UI that could offer outdated + historic timestamp when modifying timestamp again.
      • +
    +
    • +
  • Added command to toggle visibility of aircraft ahead, + available as joystick button or keyboard assignment + "LiveTraffic/Aircrafts/Toggle_Ahead" + and as new menu item "Hide aircraft ahead". +
    • +
  • OpenSky Master File: Removal of no longer used database files. +
    • +
  • Additional Help menu entries for + Support Forum and + Support HowTo. +
    • +
+ +

v3.5.0

At least copy the following files, which have changed compared to v3.4.3: