diff --git a/.github/template/fwe-build/action.yml b/.github/template/fwe-build/action.yml index d0679e5b..fc178740 100644 --- a/.github/template/fwe-build/action.yml +++ b/.github/template/fwe-build/action.yml @@ -62,7 +62,7 @@ runs: shell: bash if: inputs.build-arch == 'native' run: | - ./tools/test-fwe.sh ${{ inputs.extra-options }} --extra-ctest-args "-E \"CANDataSourceTest|ISOTPOverCANProtocolTest|IoTFleetWiseEngineTest|OBDOverCANModuleTest\"" + ./tools/test-fwe.sh ${{ inputs.extra-options }} --extra-gtest-filter "-*CANDataSourceTest*:*ISOTPOverCANProtocolTest*:*IoTFleetWiseEngineTest*:*OBDOverCANModuleTest*" - name: upload-artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4be3bca..59a7949d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: with: build-arch: "native" upload-arch: "amd64" + extra-options: "--with-greengrassv2-support" dist-name: "aws-iot-fleetwise-edge" cache-paths: /usr/local/x86_64-linux-gnu dist-files: build/aws-iot-fleetwise-edge:. @@ -49,6 +50,7 @@ jobs: with: build-arch: "cross-arm64" upload-arch: "arm64" + extra-options: "--with-greengrassv2-support" dist-name: "aws-iot-fleetwise-edge" cache-paths: /usr/local/aarch64-linux-gnu:/usr/local/x86_64-linux-gnu dist-files: build/aws-iot-fleetwise-edge:. @@ -61,7 +63,7 @@ jobs: with: build-arch: "cross-armhf" upload-arch: "armhf" - extra-options: "--with-iwave-gps-support" + extra-options: "--with-greengrassv2-support --with-iwave-gps-support" dist-name: "aws-iot-fleetwise-edge" cache-paths: /usr/local/arm-linux-gnueabihf:/usr/local/x86_64-linux-gnu dist-files: build/aws-iot-fleetwise-edge:. @@ -74,7 +76,7 @@ jobs: with: build-arch: "native" upload-arch: "amd64" - extra-options: "--with-ros2-support" + extra-options: "--with-greengrassv2-support --with-ros2-support" dist-name: "aws-iot-fleetwise-edge-ros2" cache-paths: /usr/local/x86_64-linux-gnu:/opt/ros dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:. @@ -87,7 +89,7 @@ jobs: with: build-arch: "cross-arm64" upload-arch: "arm64" - extra-options: "--with-ros2-support" + extra-options: "--with-greengrassv2-support --with-ros2-support" dist-name: "aws-iot-fleetwise-edge-ros2" cache-paths: /usr/local/aarch64-linux-gnu:/usr/local/x86_64-linux-gnu:/opt/ros dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:. @@ -100,7 +102,7 @@ jobs: with: build-arch: "cross-armhf" upload-arch: "armhf" - extra-options: "--with-ros2-support" + extra-options: "--with-greengrassv2-support --with-ros2-support" dist-name: "aws-iot-fleetwise-edge-ros2" cache-paths: /usr/local/arm-linux-gnueabihf:/usr/local/x86_64-linux-gnu:/opt/ros dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0563e86..2af869d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks default_language_version: @@ -121,10 +123,17 @@ repos: verbose: true - id: validate-build-files name: validate-build-files - entry: - bash -e -c 'FILE_MISSING=false; for FILE in "$@"; do if ! grep -q "${FILE}" - CMakeLists.txt; then echo "${FILE} is missing from CMakeLists.txt"; FILE_MISSING=true; fi; - done; if ${FILE_MISSING}; then exit -1; fi' -- + entry: | + bash -e -c ' + VIOLATION=false + for FILE in "$@"; do + if ! grep -q "${FILE}" CMakeLists.txt; then + echo "${FILE} is missing from CMakeLists.txt" + VIOLATION=true + fi + done + if ${VIOLATION}; then exit -1; fi + ' -- language: python files: "^src|^test/unit" exclude: "^test/unit/support" @@ -141,3 +150,20 @@ repos: language: pygrep entry: "[\\x80-\\xFF]" types: [text] + - id: ensure-copyright-header + name: ensure-copyright-header + entry: | + bash -e -c ' + VIOLATION=false + for FILE in "$@"; do + if ! head -n2 "${FILE}" | grep -q "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."; then + echo "${FILE} is missing copyright headers" + VIOLATION=true + fi + done + if ${VIOLATION}; then exit -1; fi + ' -- + language: python + types_or: [c++, python, yaml, cmake, proto, shell] + exclude: "gradlew|.prettierrc.yaml|.clang-format|.clang-tidy|create_avd_config.sh" + verbose: true diff --git a/CHANGELOG.md b/CHANGELOG.md index c18df677..9c2b977d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # Change Log +## v1.1.2 (2024-10-29) + +Bug fixes: + +- MQTT connection fixes: + - Retry MQTT topic subscription when it fails. When FWE starts up, it tries to establish the MQTT + connection and retries until it succeeds. Only after that, it subscribes to the topics. But if a + subscription failed (e.g. due to network issues), it never retried, making FWE never receive + messages from the topic until the process is restarted. + - On shutdown only unsubscribe to MQTT Channels that are subscribed + - When persistent sessions are enabled, don't unsubscribe on shutdown. This is required for FWE to + receive messages while it was offline. + - Connect only after all listeners are subscribed. If a message was received right after + connection was established, it could be silently ignored because the listeners for the topic + weren't registered yet. +- ROS2 related bug fixes: + - If more than one campaign used a ROS2 signal in its expression, only the first campaign would + receive data for evaluation. + - If FWE receives a campaign more than once from the cloud, data collection would stop due to + internal signal IDs being reallocated. + - Fixed segfault for VSD build when campaigns received before decoder manifest +- Fixed invalid read on shutdown when network is down + +Improvements: + +- Update AWS C++ SDK to `v1.11.284`. +- Update Boost to `1.84.0`. +- Enable flow control for MQTT5 client (when using `iotCore` connection type). This limits data sent + by FWE to what is defined in IoT Core limits for throughput and number of publishes. +- Change default values for some MQTT connection settings and make them configurable in the config + file. All defaults are now the same as the AWS SDK: + - Keep alive interval changed from `60` seconds to `1200` seconds. Set `keepAliveIntervalSeconds` + to override it. + - Ping timeout default changed from `3000` ms to `30000` ms. Set `pingTimeoutMs` to override it. + - Persistent sessions are now disabled by default. Set `sessionExpiryIntervalSeconds` to a + non-zero value to enable it. +- Add implicit casting between numeric and Boolean data types in expression evaluation. E.g. + `1 + true` will equal `2`, and `false || 3` will equal `true`. +- Add support for relative paths to the certificate, private key files and persistency directory, + relative to the directory containing the configuration JSON file. +- Make `demo.sh` more generic, by 1/ Allowing multiple 'node', 'decoder', 'network interface', and + 'campaign' JSON files to be passed, rather than having specific options for CAN, OBD and ROS2, 2/ + Add `--data-destination` option to specify data destination (default is still Amazon Timestream.) + This allows the Android-specific demo script to be removed. +- Previously if the CAN network interface goes down, FWE would exit with an error. Now it will + continue to run and will resume data collection if the interface comes back up, however it will + still exit with an error if the interface is removed from the system. +- Improved developer guides +- Added Disconnect Packet to the MQTT Client Stop sequence to help customers understand what the + reason for the vehicle disconnection from the broker +- Added usage of data type from decoder manifest for CAN and OBD signals +- Add new signal type UNKNOWN +- Split MQTT channels into separate Sender and Receiver compared to previously using the same + channel instance to send and receive messages +- Adaptive payload sizing for both compressed and uncompressed data + ## v1.1.1 (2024-02-12) Bug fixes: diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f6c041f..aaffc684 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10.2) -project(iotfleetwise VERSION 1.1.1) +project(iotfleetwise VERSION 1.1.2) # FWE uses C++14 for compatibility reasons with Automotive middlewares (Adaptive AUTOSAR, ROS2) # Note: When built with FWE_FEATURE_ROS2, colcon will override these settings @@ -29,7 +29,8 @@ option(FWE_FEATURE_CUSTOM_DATA_SOURCE "Include the custom data source interface, option(FWE_FEATURE_IWAVE_GPS "Include the IWave GPS example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF) option(FWE_FEATURE_EXTERNAL_GPS "Include the external GPS example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF) option(FWE_FEATURE_AAOS_VHAL "Include the Android Automotive VHAL example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF) -option(FWE_FEATURE_VISION_SYSTEM_DATA "Include support for vision-system-data sources" OFF) +option(FWE_FEATURE_S3 "Internal option, not useful on its own. Enables usage of AWS credential provider, and uploading and downloading from S3." OFF) +option(FWE_FEATURE_VISION_SYSTEM_DATA "Include support for vision-system-data sources. Implies FWE_FEATURE_S3." OFF) option(FWE_FEATURE_ROS2 "Include support for ROS2 as a vision-system-data source. Implies FWE_FEATURE_VISION_SYSTEM_DATA." OFF) option(FWE_BUILD_EXECUTABLE "Build the executable, otherwise build a library" ON) option(FWE_BUILD_ANDROID_SHARED_LIBRARY "Build the android shared library" OFF) @@ -57,8 +58,12 @@ if(FWE_FEATURE_ROS2) ament_package() endif() if(FWE_FEATURE_VISION_SYSTEM_DATA) + set(FWE_FEATURE_S3 ON FORCE) add_compile_options("-DFWE_FEATURE_VISION_SYSTEM_DATA;-DDECNUMDIGITS=34") endif() +if(FWE_FEATURE_S3) + add_compile_options("-DFWE_FEATURE_S3") +endif() # Define the default build type if(NOT CMAKE_BUILD_TYPE) @@ -116,8 +121,9 @@ set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-convers set(HEADER_FILES src/Assert.h src/AwsBootstrap.h - src/AwsIotChannel.h src/AwsIotConnectivityModule.h + src/AwsIotReceiver.h + src/AwsIotSender.h src/AwsSDKMemoryManager.h src/CacheAndPersist.h src/CANDataConsumer.h @@ -125,6 +131,7 @@ set(HEADER_FILES src/CANDataTypes.h src/CANDecoder.h src/CANInterfaceIDTranslator.h + src/CheckinSender.h src/Clock.h src/ClockHandler.h src/CollectionInspectionAPITypes.h @@ -138,15 +145,14 @@ set(HEADER_FILES src/DataSenderManager.h src/DataSenderManagerWorkerThread.h src/DataSenderProtoWriter.h + src/DataSenderTypes.h src/DecoderManifestIngestion.h src/EnumUtility.h src/EventTypes.h src/ExternalCANDataSource.h src/ICollectionScheme.h src/ICollectionSchemeList.h - src/ICollectionSchemeManager.h src/IConnectionTypes.h - src/IConnectivityChannel.h src/IConnectivityModule.h src/IDecoderDictionary.h src/IDecoderManifest.h @@ -161,6 +167,7 @@ set(HEADER_FILES src/ISOTPOverCANSender.h src/ISOTPOverCANSenderReceiver.h src/Listener.h + src/QueueTypes.h src/LoggingModule.h src/LogLevel.h src/MemoryUsageInfo.h @@ -171,6 +178,8 @@ set(HEADER_FILES src/OBDOverCANECU.h src/OBDOverCANModule.h src/PayloadManager.h + src/QueueTypes.h + src/RawDataManager.h src/RemoteProfiler.h src/RetryThread.h src/Schema.h @@ -178,6 +187,7 @@ set(HEADER_FILES src/Signal.h src/SignalTypes.h src/StreambufBuilder.h + src/TelemetryDataSender.h src/Thread.h src/Timer.h src/TimeTypes.h @@ -188,14 +198,15 @@ set(HEADER_FILES # Source files set(SRC_FILES src/AwsBootstrap.cpp - src/AwsIotChannel.cpp + src/AwsIotReceiver.cpp + src/AwsIotSender.cpp src/AwsIotConnectivityModule.cpp src/AwsSDKMemoryManager.cpp src/CacheAndPersist.cpp src/CANDataConsumer.cpp src/CANDataSource.cpp src/CANDecoder.cpp - src/CheckinAndPersistency.cpp + src/CheckinSender.cpp src/ClockHandler.cpp src/CollectionInspectionEngine.cpp src/CollectionInspectionWorkerThread.cpp @@ -222,9 +233,12 @@ set(SRC_FILES src/OBDOverCANECU.cpp src/OBDOverCANModule.cpp src/PayloadManager.cpp + src/Persistency.cpp + src/RawDataManager.cpp src/RemoteProfiler.cpp src/RetryThread.cpp src/Schema.cpp + src/TelemetryDataSender.cpp src/Thread.cpp src/TraceModule.cpp ${CMAKE_CURRENT_BINARY_DIR}/IoTFleetWiseVersion.cpp @@ -237,7 +251,6 @@ set(TEST_FILES test/unit/CacheAndPersistTest.cpp test/unit/CANDataSourceTest.cpp test/unit/CANDecoderTest.cpp - test/unit/CheckinAndPersistencyTest.cpp test/unit/ClockHandlerTest.cpp test/unit/CollectionInspectionEngineTest.cpp test/unit/CollectionInspectionWorkerThreadTest.cpp @@ -258,6 +271,8 @@ set(TEST_FILES test/unit/OBDDataDecoderTest.cpp test/unit/OBDOverCANModuleTest.cpp test/unit/PayloadManagerTest.cpp + test/unit/PersistencyTest.cpp + test/unit/RawDataManagerTest.cpp test/unit/RemoteProfilerTest.cpp test/unit/SchemaTest.cpp test/unit/ThreadTest.cpp @@ -292,28 +307,39 @@ if(FWE_FEATURE_CUSTOM_DATA_SOURCE) set(HEADER_FILES ${HEADER_FILES} src/CustomDataSource.h) endif() if(FWE_FEATURE_GREENGRASSV2) - set(SRC_FILES ${SRC_FILES} src/AwsGGConnectivityModule.cpp src/AwsGGChannel.cpp) - set(HEADER_FILES ${HEADER_FILES} src/AwsGGChannel.h src/AwsGGConnectivityModule.h) + set(SRC_FILES ${SRC_FILES} + src/AwsGreengrassV2ConnectivityModule.cpp + src/AwsGreengrassV2Receiver.cpp + src/AwsGreengrassV2Sender.cpp + ) + set(HEADER_FILES ${HEADER_FILES} + src/AwsGreengrassV2ConnectivityModule.h + src/AwsGreengrassV2Receiver.h + src/AwsGreengrassV2Sender.h +) +endif() +if(FWE_FEATURE_S3) + set(SRC_FILES ${SRC_FILES} src/Credentials.cpp) + set(TEST_FILES ${TEST_FILES} test/unit/CredentialsTest.cpp) + set(HEADER_FILES ${HEADER_FILES} + src/Credentials.h + src/TransferManagerWrapper.h + ) endif() if(FWE_FEATURE_VISION_SYSTEM_DATA) set(SRC_FILES ${SRC_FILES} - src/Credentials.cpp src/DataSenderIonWriter.cpp - src/RawDataManager.cpp src/S3Sender.cpp + src/VisionSystemDataSender.cpp ) set(TEST_FILES ${TEST_FILES} - test/unit/CredentialsTest.cpp test/unit/DataSenderIonWriterTest.cpp - test/unit/RawDataManagerTest.cpp test/unit/S3SenderTest.cpp ) set(HEADER_FILES ${HEADER_FILES} - src/Credentials.h src/DataSenderIonWriter.h - src/RawDataManager.h src/S3Sender.h - src/TransferManagerWrapper.h + src/VisionSystemDataSender.h ) endif() if(FWE_FEATURE_ROS2) @@ -357,7 +383,7 @@ endif() separate_arguments(FWE_AWS_SDK_EXTRA_LIBS) set(REQUIRED_AWS_SDK_COMPONENTS "core") -if(FWE_FEATURE_VISION_SYSTEM_DATA) +if(FWE_FEATURE_S3) set(REQUIRED_AWS_SDK_COMPONENTS "transfer;s3-crt;${REQUIRED_AWS_SDK_COMPONENTS}") endif() set(OLD_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) @@ -546,10 +572,10 @@ if(${BUILD_TESTING}) add_executable(ROS2DataSourceTest test/unit/ROS2DataSourceTest.cpp src/ROS2DataSource.cpp + src/RawDataManager.cpp src/Thread.cpp src/LoggingModule.cpp src/ClockHandler.cpp - src/RawDataManager.cpp src/ConsoleLogger.cpp src/IoTFleetWiseConfig.cpp test/unit/support/main.cpp diff --git a/README.md b/README.md index 93893d7a..916b592b 100644 --- a/README.md +++ b/README.md @@ -112,21 +112,21 @@ See [SECURITY](./SECURITY.md) for more information FWE depends on the following open source libraries. Refer to the corresponding links for more information. -- [AWS SDK for C++: v1.11.177](https://github.com/aws/aws-sdk-cpp) +- [AWS SDK for C++: v1.11.284](https://github.com/aws/aws-sdk-cpp) - [Curl: v7.58.0](https://github.com/curl/curl) - [OpenSSL: v1.1.1](https://github.com/openssl/openssl) - [zlib: v1.2.11](https://github.com/madler/zlib) - [GoogleTest: v1.10.0](https://github.com/google/googletest) - [Google Benchmark: v1.6.1](https://github.com/google/benchmark) - [Protobuf: v3.21.12](https://github.com/protocolbuffers/protobuf) -- [Boost: v1.78.0](https://github.com/boostorg/boost) +- [Boost: v1.84.0](https://github.com/boostorg/boost) - [JsonCpp: v1.9.5](https://github.com/open-source-parsers/jsoncpp) - [Snappy: v1.1.8](https://github.com/google/snappy) Optional: The following dependencies are only required when the option `FWE_FEATURE_GREENGRASSV2` is enabled. -- [AWS IoT Device SDK for C++ v2: v1.30.0](https://github.com/aws/aws-iot-device-sdk-cpp-v2) +- [AWS IoT Device SDK for C++ v2: v1.32.2](https://github.com/aws/aws-iot-device-sdk-cpp-v2) Optional: The following dependencies are only required when the option `FWE_FEATURE_VISION_SYSTEM_DATA` is enabled. diff --git a/THIRD-PARTY-LICENSES b/THIRD-PARTY-LICENSES index fbcd16e9..88574298 100644 --- a/THIRD-PARTY-LICENSES +++ b/THIRD-PARTY-LICENSES @@ -1,8 +1,8 @@ The Reference Implementation for AWS IoT FleetWise includes the following third-party software/licensing: -** AWS SDK for C++: v1.11.177 - https://github.com/aws/aws-sdk-cpp -** AWS IoT Device SDK for C++ v2: v1.30.0 - https://github.com/aws/aws-iot-device-sdk-cpp-v2 +** AWS SDK for C++: v1.11.284 - https://github.com/aws/aws-sdk-cpp +** AWS IoT Device SDK for C++ v2: v1.32.2 - https://github.com/aws/aws-iot-device-sdk-cpp-v2 ** Google Benchmark: v1.6.1 - https://github.com/google/benchmark ** OpenSSL: v1.1.1 - https://github.com/openssl/openssl ** ROS2: Galactic - https://github.com/ros2/rclcpp @@ -248,7 +248,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------- -** Boost: v1.78.0 - https://github.com/boostorg/boost +** Boost: v1.84.0 - https://github.com/boostorg/boost Boost Software License - Version 1.0 - August 17th, 2003 diff --git a/cmake/compiler_gcc.cmake b/cmake/compiler_gcc.cmake index ecd5bdad..1517c362 100644 --- a/cmake/compiler_gcc.cmake +++ b/cmake/compiler_gcc.cmake @@ -1,4 +1,6 @@ -add_compile_options(-Wconversion -Wall -Wextra -pedantic -ffunction-sections -fdata-sections) +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +add_compile_options(-Wconversion -Wall -Wextra -pedantic -ffunction-sections -fdata-sections -fno-omit-frame-pointer) link_libraries( -Wl,--gc-sections # Remove all unreferenced sections $<$:-Wl,-s> # Strip all symbols diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index c041bb71..6e6ac112 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + # check if Doxygen is installed find_package(Doxygen OPTIONAL_COMPONENTS dot) diff --git a/cmake/graphviz.cmake b/cmake/graphviz.cmake index 8456dde4..3f268e29 100644 --- a/cmake/graphviz.cmake +++ b/cmake/graphviz.cmake @@ -1 +1,3 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + set(GRAPHVIZ_IGNORE_TARGETS "Gtest;Test;fuzz") diff --git a/cmake/unit_test.cmake b/cmake/unit_test.cmake index 30822b96..f1b15ddb 100644 --- a/cmake/unit_test.cmake +++ b/cmake/unit_test.cmake @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + function(add_unit_test TEST_NAME) add_test( NAME ${TEST_NAME} diff --git a/cmake/valgrind.cmake b/cmake/valgrind.cmake index 926201c3..94e9c294 100644 --- a/cmake/valgrind.cmake +++ b/cmake/valgrind.cmake @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + if(FWE_VALGRIND) find_program(VALGRIND_COMMAND valgrind) if(NOT VALGRIND_COMMAND) diff --git a/configuration/static-config.json b/configuration/static-config.json index cf8eaf87..14105dfa 100644 --- a/configuration/static-config.json +++ b/configuration/static-config.json @@ -51,6 +51,9 @@ "connectionType": "iotCore", "endpointUrl": "my-endpoint.my-region.amazonaws.com", "clientId": "VEHICLE_ID_GOES_HERE", + "keepAliveIntervalSeconds": 60, + "pingTimeoutMs": 30000, + "sessionExpiryIntervalSeconds": 0, "collectionSchemeListTopic": "$aws/iotfleetwise/vehicles/VEHICLE_ID_GOES_HERE/collection_schemes", "decoderManifestTopic": "$aws/iotfleetwise/vehicles/VEHICLE_ID_GOES_HERE/decoder_manifests", "canDataTopic": "$aws/iotfleetwise/vehicles/VEHICLE_ID_GOES_HERE/signals", diff --git a/docs/dev-guide/edge-agent-dev-guide-nxp-s32g.md b/docs/dev-guide/edge-agent-dev-guide-nxp-s32g.md index 0c16b18c..5af8bed0 100644 --- a/docs/dev-guide/edge-agent-dev-guide-nxp-s32g.md +++ b/docs/dev-guide/edge-agent-dev-guide-nxp-s32g.md @@ -162,7 +162,12 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ ```bash cd ~/aws-iot-fleetwise-edge/tools/cloud \ - && ./demo.sh --vehicle-name fwdemo-s32g --campaign-file campaign-obd-heartbeat.json + && ./demo.sh \ + --vehicle-name fwdemo-s32g \ + --node-file obd-nodes.json \ + --decoder-file obd-decoders.json \ + --network-interface-file network-interface-obd.json \ + --campaign-file campaign-obd-heartbeat.json ``` ## Clean up diff --git a/docs/dev-guide/edge-agent-dev-guide-renesas-rcar-s4.md b/docs/dev-guide/edge-agent-dev-guide-renesas-rcar-s4.md index b66a3add..8af46241 100644 --- a/docs/dev-guide/edge-agent-dev-guide-renesas-rcar-s4.md +++ b/docs/dev-guide/edge-agent-dev-guide-renesas-rcar-s4.md @@ -178,7 +178,12 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ ```bash cd ~/aws-iot-fleetwise-edge/tools/cloud sudo -H ./install-deps.sh - ./demo.sh --vehicle-name fwdemo-rcars4 --campaign-file campaign-obd-heartbeat.json + ./demo.sh \ + --vehicle-name fwdemo-rcars4 \ + --node-file obd-nodes.json \ + --decoder-file obd-decoders.json \ + --network-interface-file network-interface-obd.json \ + --campaign-file campaign-obd-heartbeat.json ``` ## Clean up diff --git a/docs/dev-guide/edge-agent-dev-guide.md b/docs/dev-guide/edge-agent-dev-guide.md index 877aa75a..6a318be7 100644 --- a/docs/dev-guide/edge-agent-dev-guide.md +++ b/docs/dev-guide/edge-agent-dev-guide.md @@ -139,25 +139,36 @@ collect data from it. The above command installs the following PIP packages: `wrapt plotly pandas cantools pyarrow` -1. If you are using the AWS CLI with a version lower than v2.11.24, update the CLI by running: +1. Run the following command to generate 'node' and 'decoder' JSON files from the input DBC file: ```bash - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ - && unzip -q awscliv2.zip \ - && sudo ./aws/install --update \ - && rm -rf ./aws* + python3 dbc-to-nodes.py hscan.dbc can-nodes.json \ + && python3 dbc-to-decoders.py hscan.dbc can-decoders.json ``` 1. Run the demo script: ```bash - ./demo.sh --vehicle-name fwdemo + ./demo.sh \ + --vehicle-name fwdemo \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json ``` - - (Optional) To enable S3 upload, append the option `--enable-s3-upload` + - (Optional) To enable S3 upload, append the option `--data-destination S3`. By default the + upload format will be JSON. You can change this to Parquet format for S3 by passing + `--s3-format PARQUET`. ```bash - ./demo.sh --vehicle-name fwdemo --enable-s3-upload + ./demo.sh \ + --vehicle-name fwdemo \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json \ + --data-destination S3 ``` - (Optional) If you selected a `FleetSize` of greater than one above, append the option @@ -171,7 +182,14 @@ collect data from it. `myfwdemo`, you must pass those values when calling `demo.sh`: ```bash - ./demo.sh --vehicle-name myfwdemo --fleet-size 2 --region eu-central-1 + ./demo.sh \ + --vehicle-name myfwdemo \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json \ + --fleet-size 2 \ + --region eu-central-1 ``` The demo script: @@ -179,14 +197,11 @@ collect data from it. 1. Registers your AWS account with AWS IoT FleetWise, if not already registered. 1. Creates an Amazon Timestream database and table. 1. Creates IAM role and policy required for the service to write data to Amazon Timestream. - 1. Creates a signal catalog, firstly based on `obd-nodes.json` to add standard OBD signals, and - secondly based on the DBC file `hscan.dbc` to add CAN signals in a flat signal list. - 1. Creates a model manifest that references the signal catalog with all of the OBD and DBC - signals. + 1. Creates a signal catalog based on `can-nodes.json`. + 1. Creates a model manifest that references the signal catalog with all of the CAN signals. 1. Activates the model manifest. - 1. Creates a decoder manifest linked to the model manifest using `obd-decoders.json` for decoding - OBD signals from the network interfaces defined in `network-interfaces.json`. - 1. Imports the CAN signal decoding information from `hscan.dbc` to the decoder manifest. + 1. Creates a decoder manifest linked to the model manifest using `can-decoders.json` for decoding + signals from the network interfaces defined in `network-interfaces-can.json`. 1. Updates the decoder manifest to set the status as `ACTIVE`. 1. Creates a vehicle with a name equal to `fwdemo` which is the same as the name given to the CloudFormation Stack name in the previous section. @@ -205,8 +220,8 @@ collect data from it. 1. Create an S3 bucket with a bucket policy that allows AWS IoT FleetWise to write data to the bucket. - 1. Creates 2 additional campaigns from `campaign-brake-event.json`. One campaign will upload data - to to S3 in JSON format, one to S3 in parquet format. + 1. Creates an additional campaign from `campaign-brake-event.json` to upload the data to S3 in + JSON format, or Parquet format if the `--s3-format PARQUET` option is passed. 1. Wait 20 minutes for the data to propagate to S3 and then download it. 1. Save the data to an HTML file. @@ -418,31 +433,42 @@ collect data from it. The above command installs the following Ubuntu packages: `python3 python3-pip`. It then installs the following PIP packages: `wrapt plotly pandas cantools pyarrow` -1. If you are using the AWS CLI with a version lower than v2.11.24, update the CLI by running: +1. Run the following to explore the AWS IoT FleetWise CLI: ```bash - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ - && unzip -q awscliv2.zip \ - && sudo ./aws/install --update \ - && rm -rf ./aws* + aws iotfleetwise help ``` -1. Run the following to explore the AWS IoT FleetWise CLI: +1. Run the following command to generate 'node' and 'decoder' JSON files from the input DBC file. ```bash - aws iotfleetwise help + python3 dbc-to-nodes.py hscan.dbc can-nodes.json \ + && python3 dbc-to-decoders.py hscan.dbc can-decoders.json ``` 1. Run the demo script: ```bash - ./demo.sh --vehicle-name fwdemo-ec2 + ./demo.sh \ + --vehicle-name fwdemo-ec2 \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json ``` - - (Optional) To enable S3 upload, append the option `--enable-s3-upload` + - (Optional) To enable S3 upload, append the option `--data-destination S3`. By default the + upload format will be JSON. You can change this to Parquet format by passing + `--s3-format PARQUET`. ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --enable-s3-upload + ./demo.sh \ + --vehicle-name fwdemo-ec2 \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json \ + --data-destination S3 ``` - (Optional) If you changed the `--region` option to `provision.sh` above, append the option @@ -451,7 +477,13 @@ collect data from it. in the demo.sh file. ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --region eu-central-1 + ./demo.sh \ + --vehicle-name fwdemo-ec2 \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file campaign-brake-event.json \ + --region eu-central-1 ``` The demo script: @@ -459,14 +491,11 @@ collect data from it. 1. Registers your AWS account with AWS IoT FleetWise, if not already registered. 1. Creates an Amazon Timestream database and table. 1. Creates IAM role and policy required for the service to write data to Amazon Timestream. - 1. Creates a signal catalog, firstly based on `obd-nodes.json` to add standard OBD signals, and - secondly based on the DBC file `hscan.dbc` to add CAN signals in a flat signal list. - 1. Creates a model manifest that references the signal catalog with all of the OBD and DBC - signals. + 1. Creates a signal catalog based on `can-nodes.json`. + 1. Creates a model manifest that references the signal catalog with all of the CAN signals. 1. Activates the model manifest. - 1. Creates a decoder manifest linked to the model manifest using `obd-decoders.json` for decoding - OBD signals from the network interfaces defined in `network-interfaces.json`. - 1. Imports the CAN signal decoding information from `hscan.dbc` to the decoder manifest. + 1. Creates a decoder manifest linked to the model manifest using `can-decoders.json` for decoding + signals from the network interfaces defined in `network-interfaces-can.json`. 1. Updates the decoder manifest to set the status as `ACTIVE`. 1. Creates a vehicle with a name equal to `fwdemo-ec2`, the same as the name passed to `provision.sh`. @@ -485,8 +514,8 @@ collect data from it. 1. Create an S3 bucket with a bucket policy that allows AWS IoT FleetWise to write data to the bucket. - 1. Creates 2 additional campaigns from `campaign-brake-event.json`. One campaign will upload data - to to S3 in JSON format, one to S3 in parquet format. + 1. Creates an additional campaign from `campaign-brake-event.json` to upload the data to S3 in + JSON format, or Parquet format if the `--s3-format PARQUET` option is passed. 1. Wait 20 minutes for the data to propagate to S3 and then download it. 1. Save the data to an HTML file. @@ -512,14 +541,12 @@ collect data from it. data from the vehicle. Repeat the process above to view the collected data. ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --campaign-file campaign-obd-heartbeat.json - ``` - - Similarly, if you chose to deploy the 'heartbeat' campaign that collects OBD data from an AWS IoT - thing created in in Europe (Frankfurt), you must configure `--region`: - - ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --campaign-file campaign-obd-heartbeat.json --region eu-central-1 + ./demo.sh \ + --vehicle-name fwdemo-ec2 \ + --node-file obd-nodes.json \ + --decoder-file obd-decoders.json \ + --network-interface-file network-interface-obd.json \ + --campaign-file campaign-obd-heartbeat.json ``` 1. Run the following _on the development machine_ to import your custom DBC file. You also have to @@ -527,14 +554,14 @@ collect data from it. have to test data collection with the real vehicle or custom simulator. ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --dbc-file --campaign-file - ``` - - Similarly, if you chose to deploy the 'heartbeat' campaign that collects OBD data from an AWS IoT - thing created in in Europe (Frankfurt), you must configure `--region`: - - ```bash - ./demo.sh --vehicle-name fwdemo-ec2 --dbc-file --campaign-file --region eu-central-1 + python3 dbc-to-nodes.py can-nodes.json \ + && python3 dbc-to-decoders.py can-decoders.json \ + && ./demo.sh \ + --vehicle-name fwdemo-ec2 \ + --node-file can-nodes.json \ + --decoder-file can-decoders.json \ + --network-interface-file network-interface-can.json \ + --campaign-file ``` ### Clean up @@ -720,7 +747,7 @@ The code base of the Reference Implementation consists of C++ modules that imple functionalities of the layers described above. All these modules are compiled into a POSIX user space application running a single process. -**BSP Modules** +#### BSP Modules ``` CacheAndPersist @@ -744,7 +771,7 @@ These modules include a set of APIs and utility functions that the rest of the s These modules are used uniformly by all the other modules in the application. -**Vehicle Network / Middleware Management Modules** +#### Vehicle Network / Middleware Management Modules ``` ISOTPOverCANReceiver @@ -766,11 +793,10 @@ CAN Frames. These modules abstract away all the Socket and Linux networking deta the system, and expose only a circular buffer for each network interface configured, exposing the raw CAN Frames to be consumed by the Data Inspection Modules. -**Data Management Modules** +#### Data Management Modules ``` CANDecoder -CheckinAndPersistency CollectionSchemeIngestion CollectionSchemeIngestionList CollectionSchemeManager @@ -781,6 +807,7 @@ DecoderDictionaryExtractor DecoderManifestIngestion InspectionMatrixExtractor OBDDataDecoder +Persistency RawDataManager Schema ``` @@ -808,7 +835,7 @@ module will attempt to send the data to the cloud. The `CacheAndPersist` module disk space is not exhausted, so this module just invokes the persistency module when it wants to read or write data to disk. -**Data Inspection Modules** +#### Data Inspection Modules ``` CANDataConsumer @@ -836,14 +863,16 @@ Upon fulfillment of one or more trigger conditions, these modules extract from t buffer a data snapshot that's shared with the offboard connectivity modules for further processing. Again here a circular buffer is used as a transport mechanism of the data. -**Offboard Connectivity Modules** +#### Offboard Connectivity Modules ``` AwsBootstrap -AwsGGChannel -AwsGGConnectivityModule -AwsIotChannel +AwsGreengrassV2ConnectivityModule +AwsGreengrassV2Receiver +AwsGreengrassV2Sender AwsIotConnectivityModule +AwsIotReceiver +AwsIotSender AwsSDKMemoryManager Credentials PayloadManager @@ -865,7 +894,20 @@ eventual updates. On the subscribe side, these modules notifies the rest of the arrival of an update of either the Scheme or the decoder manifests, which are enacted accordant in near real time. -**Execution Management Module** +Some MQTT connection parameters can be configured in the config file (refer to +[Configuration](#configuration)). In general, FWE adopts the same defaults as the AWS SDK: + +1. Keep alive interval default is `1200` seconds. Set `keepAliveIntervalSeconds` to override it. +1. Ping timeout default is `30000` ms. Set `pingTimeoutMs` to override it. +1. Persistent sessions are disabled by default. Set `sessionExpiryIntervalSeconds` to a non-zero + value to enable it. + +Note that `keepAliveIntervalSeconds` and `pingTimeoutMs` influence how a disconnection is detected. +In case the network is down, you can expect the MQTT client to take at least the keep alive interval +plus the ping timeout to detect the interruption. For example, if you set `keepAliveIntervalSeconds` +to `60` and `pingTimeoutMs` to `3000`, it could take up to `63` seconds for the client to detect it. + +#### Execution Management Module ``` IoTFleetWiseEngine @@ -1078,6 +1120,9 @@ described below in the configuration section. Each log entry includes the follow | | rootCA | The path to the root CA certificate file (optional, either `rootCAFilename` or `rootCA` can be provided) | string | | | metricsUploadTopic | Topic used to upload application metrics in plain json. Only used if `remoteProfilerDefaultValues` section is configured | string | | | loggingUploadTopic | Topic used to upload log messages in plain json. Only used if `remoteProfilerDefaultValues` section is configured | string | +| | keepAliveIntervalSeconds | Only valid for 'iotCore' connection. The maximum time interval, in seconds, that is permitted to elapse between the point at which the client finishes transmitting one MQTT packet and the point it starts sending the next. Please note that the actual value is negotiated with the server and might be larger than the one configured. | string | +| | pingTimeoutMs | Only valid for 'iotCore' connection. Time interval to wait after sending a PINGREQ for a PINGRESP to arrive. If one does not arrive, the client will close the current connection. | string | +| | sessionExpiryIntervalSeconds | Only valid for 'iotCore' connection. The duration of persistent sessions. If omitted or set to 0, persistent sessions will be disabled. | string | | remoteProfilerDefaultValues | loggingUploadLevelThreshold | Only log messages with this or higher severity will be uploaded | integer | | | metricsUploadIntervalMs | The interval in milliseconds to wait for uploading new values of all metrics | integer | | | loggingUploadMaxWaitBeforeUploadMs | The maximum time in milliseconds to cache log messages before uploading them | string | diff --git a/docs/dev-guide/vision-system-data/vision-system-data-demo.ipynb b/docs/dev-guide/vision-system-data/vision-system-data-demo.ipynb index 310d81e3..aa8caab7 100644 --- a/docs/dev-guide/vision-system-data/vision-system-data-demo.ipynb +++ b/docs/dev-guide/vision-system-data/vision-system-data-demo.ipynb @@ -262,9 +262,18 @@ "\n", "CRED_STACK_NAME=\"${PREFIX}-FwCredentialProviderStack\"\n", "CRED_ROLE_NAME=\"${PREFIX}-provider-role\"\n", - "S3_BUCKET_NAME=`echo \"${PREFIX}-bucket\" | tr '[:upper:]' '[:lower:]'`\n", "\n", "REGION='us-east-1'\n", + "\n", + "S3_BUCKET_NAME=\"\"\n", + "if [[ -z \"${VSD_DEMO_S3_BUCKET_ENV}\" ]]; then\n", + " S3_BUCKET_NAME=\"${PREFIX}-bucket\"\n", + "else\n", + " S3_BUCKET_NAME=\"${VSD_DEMO_S3_BUCKET_ENV}\"\n", + "fi\n", + "\n", + "S3_BUCKET_NAME=`echo \"${S3_BUCKET_NAME}\" | tr '[:upper:]' '[:lower:]'`\n", + "\n", "SERVICE_PRINCIPAL=\"iotfleetwise.amazonaws.com\"\n", "\n", "SIGNAL_CATALOG_NAME=\"FW-VSD-ROS2-signal-catalog\"\n", @@ -389,7 +398,7 @@ "source": [ "## Set up the S3 bucket policy\n", "\n", - "If you haven't yet created a bucket for this demo, you can run the following cell to create it:" + "Create the S3 bucket if it doesn't exist already:" ] }, { @@ -400,7 +409,13 @@ }, "outputs": [], "source": [ - "aws s3 mb s3://$S3_BUCKET_NAME --region $REGION" + "BUCKET_LIST=$( aws s3 ls )\n", + "if grep -q \"$S3_BUCKET_NAME\" <<< \"$BUCKET_LIST\"; then\n", + " echo \"S3 bucket already exists\"\n", + "else\n", + " echo \"Creating S3 bucket...\"\n", + " aws s3 mb s3://$S3_BUCKET_NAME --region $REGION\n", + "fi\n" ] }, { @@ -707,9 +722,34 @@ " * Click on the 'Layouts' tab on the left.\n", " * Click on the 'Import layout' button.\n", " * Select the file in this folder `vision-system-data-foxglove-layout.json`.\n", - " * Click the 'Play' button to playback the recording.\n", - "\n", - "\n", + " * Click the 'Play' button to playback the recording." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install necessary python libraries to run the demo scripts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, + "outputs": [], + "source": [ + "cd ~/aws-iot-fleetwise-edge\n", + "sudo -H ./tools/cloud/install-deps.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "## Creating the necessary files" ] }, @@ -2089,35 +2129,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"spoolingMode\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"TO_DISK\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"collectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"timeBasedCollectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"periodMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"signalsToCollect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Cameras.Front.Image\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Speed\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Airbag.CollisionIntensity\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Acceleration\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", - "\u001b[1;39m}\u001b[0m" + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"spoolingMode\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"TO_DISK\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"collectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"timeBasedCollectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"periodMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"signalsToCollect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Cameras.Front.Image\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Speed\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Airbag.CollisionIntensity\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Acceleration\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + "\u001b[1;39m}\u001b[0m" ] } ], @@ -2205,48 +2245,48 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"spoolingMode\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"TO_DISK\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"collectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"timeBasedCollectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"periodMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"signalsToCollect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Cameras.Front.Image\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Speed\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Airbag.CollisionIntensity\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Acceleration\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"signalCatalogArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:iotfleetwise:::signal-catalog/\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"targetArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:iotfleetwise:::fleet/\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"dataDestinationConfigs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"s3Config\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"bucketArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:s3:::\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"prefix\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"-s3\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"dataFormat\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"PARQUET\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"storageCompressionFormat\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"NONE\"\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", - "\u001b[1;39m}\u001b[0m" + "\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"spoolingMode\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"TO_DISK\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"collectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"timeBasedCollectionScheme\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"periodMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"signalsToCollect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Cameras.Front.Image\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Speed\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Airbag.CollisionIntensity\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Vehicle.Acceleration\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"maxSampleCount\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m10\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"minimumSamplingIntervalMs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39m1000\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"signalCatalogArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:iotfleetwise:::signal-catalog/\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"targetArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:iotfleetwise:::fleet/\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"dataDestinationConfigs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", + " \u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"s3Config\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", + " \u001b[0m\u001b[34;1m\"bucketArn\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"arn:aws:s3:::\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"prefix\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"-s3\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"dataFormat\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"PARQUET\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0m\u001b[34;1m\"storageCompressionFormat\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"NONE\"\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", + " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", + "\u001b[1;39m}\u001b[0m" ] } ], @@ -2261,7 +2301,7 @@ " | jq \".dataDestinationConfigs=[{ \\\n", " s3Config:{ \\\n", " bucketArn:\\\"arn:aws:s3:::${S3_BUCKET_NAME}\\\", \\\n", - " prefix:\\\"${CAMPAIGN_NAME}-s3\\\", \\\n", + " prefix:\\\"${DISAMBIGUATOR}-${CAMPAIGN_NAME}-s3\\\", \\\n", " dataFormat:\\\"${DATA_FORMAT}\\\", \\\n", " storageCompressionFormat:\\\"NONE\\\" \\\n", " } \\\n", @@ -2419,7 +2459,7 @@ }, "outputs": [], "source": [ - "S3_URL=\"s3://${S3_BUCKET_NAME}/${CAMPAIGN_NAME}-s3/processed-data/\"\n", + "S3_URL=\"s3://${S3_BUCKET_NAME}/${DISAMBIGUATOR}-${CAMPAIGN_NAME}-s3/processed-data/\"\n", "\n", "while ! COLLECTED_FILES=`aws s3 ls --recursive ${S3_URL}`; do\n", " sleep 60\n", @@ -2707,7 +2747,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Delete the signal catalog" + "### (Optional) Delete the signal catalog\n", + "\n", + "Note: To delete the signal catalog, uncomment the following commands and execute them." ] }, { @@ -2718,9 +2760,9 @@ }, "outputs": [], "source": [ - "echo \"Deleting signal catalog...\"\n", - "aws ${ENDPOINT_URL_OPTION} --region $REGION iotfleetwise delete-signal-catalog \\\n", - " --name ${SIGNAL_CATALOG_NAME}" + "# echo \"Deleting signal catalog...\"\n", + "# aws ${ENDPOINT_URL_OPTION} --region $REGION iotfleetwise delete-signal-catalog \\\n", + "# --name ${SIGNAL_CATALOG_NAME}" ] }, { @@ -2764,11 +2806,11 @@ }, "outputs": [], "source": [ + "# echo \"Clean up the S3 bucket...\"\n", "# Uncomment to delete the folder\n", - "#aws s3 rm s3://${S3_BUCKET_NAME}/${CAMPAIGN_NAME}-s3 --recursive --quiet --region $REGION\n", - "\n", + "# aws s3 rm s3://${S3_BUCKET_NAME}/${DISAMBIGUATOR}-${CAMPAIGN_NAME}-s3 --recursive --quiet --region $REGION\n", "# Uncomment to delete the bucket itself\n", - "#aws s3api delete-bucket --bucket ${S3_BUCKET_NAME} --region $REGION" + "# aws s3api delete-bucket --bucket ${S3_BUCKET_NAME} --region $REGION" ] }, { diff --git a/docs/iwave-g26-tutorial/iwave-g26-tutorial.md b/docs/iwave-g26-tutorial/iwave-g26-tutorial.md index 6c26d9e0..83fa15d0 100644 --- a/docs/iwave-g26-tutorial/iwave-g26-tutorial.md +++ b/docs/iwave-g26-tutorial/iwave-g26-tutorial.md @@ -494,7 +494,12 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ commands: ```bash - ./demo.sh --vehicle-name fwdemo-g26 --campaign-file campaign-obd-heartbeat.json + ./demo.sh \ + --vehicle-name fwdemo-g26 \ + --node-file obd-nodes.json \ + --decoder-file obd-decoders.json \ + --network-interface-file network-interface-obd.json \ + --campaign-file campaign-obd-heartbeat.json ``` The demo script: @@ -502,14 +507,11 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ 1. Registers your AWS account with AWS IoT FleetWise, if not already registered. 1. Creates an Amazon Timestream database and table. 1. Creates IAM role and policy required for the service to write data to Amazon Timestream. - 1. Creates a signal catalog, firstly based on `obd-nodes.json` to add standard OBD signals, and - secondly based on the DBC file `hscan.dbc` to add CAN signals in a flat signal list. - 1. Creates a model manifest that references the signal catalog with all of the OBD and DBC - signals. + 1. Creates a signal catalog based on `obd-nodes.json`. + 1. Creates a model manifest that references the signal catalog with all of the OBD signals. 1. Activates the model manifest. 1. Creates a decoder manifest linked to the model manifest using `obd-decoders.json` for decoding - OBD signals from the network interfaces defined in `network-interfaces.json`. - 1. Imports the CAN signal decoding information from `hscan.dbc` to the decoder manifest. + signals from the network interfaces defined in `network-interfaces-obd.json`. 1. Updates the decoder manifest to set the status as `ACTIVE`. 1. Creates a vehicle with a name equal to `fwdemo-g26`, the same as the name passed to `provision.sh`. diff --git a/docs/metrics.md b/docs/metrics.md index da743cdb..1b7943d7 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -12,11 +12,11 @@ entire log of all FWE instances running. - **`ConInt`** and **`ConRes`** enable you to monitor the the number of MQTT connection interruptions and connection resumptions. If and how long it takes to detect a connection loss depends on the kernel configuration parameters `/proc/sys/net/ipv4/tcp/keepalive*` and the compile - time constants of FWE: `MQTT_CONNECT_KEEP_ALIVE_SECONDS` and `MQTT_PING_TIMEOUT_MS`. If the values - of the metric `ConInt` are not null, the internet coverage in the tested environment might be - unreliable, or `MQTT_PING_TIMEOUT_MS`, which defaults to 3 seconds, needs to be increased because - there's high latency to the IoT Core endpoint. Changing the AWS Region can help to decrease - latency. + time constants of FWE: `MQTT_KEEP_ALIVE_INTERVAL_SECONDS` and `MQTT_PING_TIMEOUT_MS`. If the + values of the metric `ConInt` are not null, the internet coverage in the tested environment might + be unreliable, or `MQTT_PING_TIMEOUT_MS`, which defaults to 3 seconds, needs to be increased + because there's high latency to the IoT Core endpoint. Changing the AWS Region can help to + decrease latency. - **`CeTrgCnt`** is a monotonic counter that monitors the number of triggers (inspection rules) detected since the FWE process started. Triggers are detected if one or more data collection campaign conditions are true. If this counter is larger than zero, but no data appears in the @@ -24,8 +24,8 @@ entire log of all FWE instances running. bus activity), or the data has been ingested to the cloud but there was an error processing it. To debug this, [enable cloud logs in AWS IoT Fleetwise settings](https://docs.aws.amazon.com/iot-fleetwise/latest/developerguide/logging-cw.html). -- **`QUEUE_CONSUMER_TO_INSPECTION_SIGNALS`** monitors the current count of signals in queue to the - signal history buffer. If this value is close to the value defined in the static config +- **`QueueConsumerToInspectionDataFrames`** monitors the current count of data frames in queue to + the signal history buffer. If this value is close to the value defined in the static config `decodedSignalsBufferSize`, increase the static config, decrease `inspectionThreadIdleTimeMs`, reduce the bus load or reduce the amount of decoded signals in the decoder manifest in the cloud. - **`ConRej`** monitors the number of MQTT connection rejects. If this is not zero check the diff --git a/docs/rpi-tutorial/raspberry-pi-tutorial.md b/docs/rpi-tutorial/raspberry-pi-tutorial.md index a3b7fa39..25e7ebdb 100644 --- a/docs/rpi-tutorial/raspberry-pi-tutorial.md +++ b/docs/rpi-tutorial/raspberry-pi-tutorial.md @@ -122,6 +122,33 @@ to take action based on your use of AWS IoT FleetWise._** ``` 1. Save the file (`CTRL+O`, `CTRL+X`) and reboot the Raspberry Pi (`sudo reboot`). +1. The Raspberry Pi can be used to collect data from a real vehicle, by connecting it via an + [OBD-II plug connector](https://www.amazon.com/s?k=obd-ii+plug+open+wire), or you can also follow + these steps to setup a CAN simulator also on the Raspberry Pi to supply CAN data to FWE: + 1. Physically connect the `CAN0` and `CAN1` interfaces together on the CAN Bus Expansion HAT + using two pieces of wire. (This will allow CAN messages to be sent on the `can0` interface, + otherwise without physical acknowledgements nothing can be sent.) + - Connect `CAN0` `H` to `CAN1` `H` + - Connect `CAN0` `L` to `CAN1` `L` + 1. On the Raspberry Pi, install the simulator dependencies: + ```bash + sudo apt update + sudo apt install -y python3 python3-pip + pip3 install \ + wrapt==1.10.0 \ + cantools==36.4.0 \ + prompt_toolkit==3.0.21 \ + python-can==3.3.4 \ + can-isotp==1.7 \ + matplotlib==3.4.3 + ``` + 1. Run the simulator on the `can0` interface, and leave this terminal open while you continue + with the remaining steps: + ```bash + cd ~/aws-iot-fleetwise-deploy/tools/cansim + python3 cansim.py -i can0 + ``` + ## Step 2: Launch your development machine These steps require an Ubuntu 20.04 development machine with 10 GB free disk space. If necessary, @@ -231,8 +258,8 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ ``` 1. As described in step 4 of [setting up the Raspberry Pi](#step-1-setup-the-raspberry-pi), connect - through SSH to the Raspberry Pi. On the Raspberry Pi, install your Edge Agent as a service by - running the following command: + through SSH to the Raspberry Pi in a new terminal. On the Raspberry Pi, install your Edge Agent + as a service by running the following command: ```bash mkdir -p ~/aws-iot-fleetwise-deploy && cd ~/aws-iot-fleetwise-deploy \ @@ -288,7 +315,12 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ commands: ```bash - ./demo.sh --vehicle-name fwdemo-rpi --campaign-file campaign-obd-heartbeat.json + ./demo.sh \ + --vehicle-name fwdemo-rpi \ + --node-file obd-nodes.json \ + --decoder-file obd-decoders.json \ + --network-interface-file network-interface-obd.json \ + --campaign-file campaign-obd-heartbeat.json ``` The demo script: @@ -296,14 +328,11 @@ mkdir -p ~/aws-iot-fleetwise-deploy \ 1. Registers your AWS account with AWS IoT FleetWise, if not already registered. 1. Creates an Amazon Timestream database and table. 1. Creates IAM role and policy required for the service to write data to Amazon Timestream. - 1. Creates a signal catalog, firstly based on `obd-nodes.json` to add standard OBD signals, and - secondly based on the DBC file `hscan.dbc` to add CAN signals in a flat signal list. - 1. Creates a model manifest that references the signal catalog with all of the OBD and DBC - signals. + 1. Creates a signal catalog based on `obd-nodes.json`. + 1. Creates a model manifest that references the signal catalog with all of the OBD signals. 1. Activates the model manifest. 1. Creates a decoder manifest linked to the model manifest using `obd-decoders.json` for decoding - OBD signals from the network interfaces defined in `network-interfaces.json`. - 1. Imports the CAN signal decoding information from `hscan.dbc` to the decoder manifest. + signals from the network interfaces defined in `network-interface-obd.json`. 1. Updates the decoder manifest to set the status as `ACTIVE`. 1. Creates a vehicle with a name equal to `fwdemo-rpi`, the same as the name passed to `provision.sh`. diff --git a/interfaces/persistency/examples/persistencyMetadataFormat.json b/interfaces/persistency/examples/persistencyMetadataFormat.json index f1938bf5..bd6e6642 100644 --- a/interfaces/persistency/examples/persistencyMetadataFormat.json +++ b/interfaces/persistency/examples/persistencyMetadataFormat.json @@ -12,25 +12,11 @@ "compressionRequired": true }, { - "filename": "exampleFile3.10n", - "payloadSize": 3428, - "compressionRequired": false, - "s3UploadMetadata": { - "bucketName": "example-bucket-name", - "bucketOwner": "012345678901", - "region": "eu-central-1" - } - }, - { - "filename": "exampleFile4.10n", - "payloadSize": 245, - "compressionRequired": false, - "s3UploadMetadata": { - "bucketName": "example-bucket-name", - "bucketOwner": "012345678901", - "region": "eu-central-1", - "uploadID": "VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR", - "partNumber": 1 + "type": "Telemetry", + "payload": { + "filename": "exampleFile3.bin", + "payloadSize": 245, + "compressionRequired": false } } ] diff --git a/interfaces/persistency/schemas/persistencyMetadataFormat.json b/interfaces/persistency/schemas/persistencyMetadataFormat.json index 8a994560..4810ef48 100644 --- a/interfaces/persistency/schemas/persistencyMetadataFormat.json +++ b/interfaces/persistency/schemas/persistencyMetadataFormat.json @@ -15,50 +15,90 @@ "additionalProperties": false, "description": "Persisted payload filenames with associated metadata", "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "filename": { - "type": "string", - "description": "Payload filename" - }, - "payloadSize": { - "type": "number", - "description": "Payload size is used to validate that payload was persisted completely" - }, - "compressionRequired": { - "type": "boolean", - "description": "Specifies if payload compression was required by campaign" - }, - "s3UploadMetadata": { + "anyOf": [ + { "type": "object", + "description": "Legacy payload metadata for Telemetry. This is only produced by older versions.", "additionalProperties": false, "properties": { - "bucketName": { + "filename": { "type": "string", - "description": "S3 bucket name set in campaign" + "description": "Payload filename" }, - "bucketOwner": { - "type": "string", - "description": "Account ID of the bucket owner" + "payloadSize": { + "type": "number", + "description": "Payload size is used to validate that payload was persisted completely" }, - "region": { - "type": "string", - "description": "Region of S3 bucket set in campaign" + "compressionRequired": { + "type": "boolean", + "description": "Specifies if payload compression was required by campaign" }, - "uploadID": { + "s3UploadMetadata": { + "type": "object", + "additionalProperties": false, + "properties": { + "bucketName": { + "type": "string", + "description": "S3 bucket name set in campaign" + }, + "bucketOwner": { + "type": "string", + "description": "Account ID of the bucket owner" + }, + "region": { + "type": "string", + "description": "Region of S3 bucket set in campaign" + }, + "uploadID": { + "type": "string", + "description": "Upload ID for multipart upload" + }, + "partNumber": { + "type": "number", + "description": "ID of multipart for multipart upload" + } + }, + "required": ["bucketName", "region"] + } + }, + "required": ["filename", "payloadSize", "compressionRequired"] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "type": "string", - "description": "Upload ID for multipart upload" + "description": "Specifies the type of this payload", + "enum": ["Telemetry"] }, - "partNumber": { - "type": "number", - "description": "ID of multipart for multipart upload" + "payload": { + "anyOf": [ + { + "type": "object", + "description": "Payload metadata for Telemetry data", + "additionalProperties": false, + "properties": { + "filename": { + "type": "string", + "description": "Payload filename" + }, + "payloadSize": { + "type": "number", + "description": "Payload size is used to validate that payload was persisted completely" + }, + "compressionRequired": { + "type": "boolean", + "description": "Specifies if payload compression was required by campaign" + } + }, + "required": ["filename", "payloadSize", "compressionRequired"] + } + ] } - }, - "required": ["bucketName", "region"] + } } - }, - "required": ["filename", "payloadSize", "compressionRequired"] + ] } } }, diff --git a/interfaces/protobuf/schemas/cloudToEdge/decoder_manifest.proto b/interfaces/protobuf/schemas/cloudToEdge/decoder_manifest.proto index 2c318fe2..d4723150 100644 --- a/interfaces/protobuf/schemas/cloudToEdge/decoder_manifest.proto +++ b/interfaces/protobuf/schemas/cloudToEdge/decoder_manifest.proto @@ -39,7 +39,8 @@ message CANSignal { /* * Unique integer identifier of the signal generated by Cloud Designer - * The most significant bit is reserved for internal usage + * Value zero is reserved for internal usage. + * The most significant bit is reserved for internal usage. */ uint32 signal_id = 1; @@ -99,7 +100,8 @@ message OBDPIDSignal { /* * Unique numeric identifier for the OBDII-PID signal - * The most significant bit is reserved for internal usage + * Value zero is reserved for internal usage. + * The most significant bit is reserved for internal usage. */ uint32 signal_id = 1; @@ -253,13 +255,13 @@ message ComplexType { } } - message ComplexSignal { /* * Unique integer identifier of the signal across all types ( i.e. CANSignal, OBDPIDSignal, ComplexSignal) * As this signal_id refers to a complex type, when referenced in places where a primitive type is required it * has to be accompanied with a path that refers to a primitive type in a PrimitiveData leaf in root_type_id. - * The most significant bit is reserved for internal usage + * Value zero is reserved for internal usage. + * The most significant bit is reserved for internal usage. */ uint32 signal_id = 1; diff --git a/interfaces/protobuf/schemas/edgeConfiguration/staticConfiguration.json b/interfaces/protobuf/schemas/edgeConfiguration/staticConfiguration.json index 27474608..070ed979 100644 --- a/interfaces/protobuf/schemas/edgeConfiguration/staticConfiguration.json +++ b/interfaces/protobuf/schemas/edgeConfiguration/staticConfiguration.json @@ -306,6 +306,18 @@ "type": "string", "description": "The root CA certificate (optional)" }, + "keepAliveIntervalSeconds": { + "type": "integer", + "description": "Only valid for 'iotCore' connection. The maximum time interval, in seconds, that is permitted to elapse between the point at which the client finishes transmitting one MQTT packet and the point it starts sending the next." + }, + "pingTimeoutMs": { + "type": "integer", + "description": "Only valid for 'iotCore' connection. Time interval to wait after sending a PINGREQ for a PINGRESP to arrive. If one does not arrive, the client will close the current connection." + }, + "sessionExpiryIntervalSeconds": { + "type": "integer", + "description": "Only valid for 'iotCore' connection. The duration of persistent sessions." + }, "connectionType": { "type": "string", "enum": ["iotCore", "iotGreengrassV2"], diff --git a/src/AaosVhalSource.cpp b/src/AaosVhalSource.cpp index 8f7203c4..0b4942cd 100644 --- a/src/AaosVhalSource.cpp +++ b/src/AaosVhalSource.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include "AaosVhalSource.h" -#include "LoggingModule.h" -#include "TraceModule.h" +#include "CollectionInspectionAPITypes.h" +#include "QueueTypes.h" #include namespace Aws @@ -14,8 +14,8 @@ namespace IoTFleetWise // NOLINT below due to C++17 warning of redundant declarations that are required to maintain C++14 compatibility constexpr const char *AaosVhalSource::CAN_CHANNEL_NUMBER; // NOLINT constexpr const char *AaosVhalSource::CAN_RAW_FRAME_ID; // NOLINT -AaosVhalSource::AaosVhalSource( SignalBufferPtr signalBufferPtr ) - : mSignalBufferPtr{ std::move( signalBufferPtr ) } +AaosVhalSource::AaosVhalSource( SignalBufferDistributorPtr signalBufferDistributor ) + : mSignalBufferDistributor{ std::move( signalBufferDistributor ) } { } @@ -55,21 +55,14 @@ AaosVhalSource::getVehiclePropertyInfo() } void -AaosVhalSource::setVehicleProperty( SignalID signalId, double value ) +AaosVhalSource::setVehicleProperty( SignalID signalId, const DecodedSignalValue &value ) { + auto signalType = SignalType::DOUBLE; auto timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( signalId, timestamp, value ) ); + collectedSignalsGroup.push_back( CollectedSignal::fromDecodedSignal( signalId, timestamp, value, signalType ) ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - - if ( !mSignalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS ); - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - FWE_LOG_WARN( "Signal buffer full" ); - } + mSignalBufferDistributor->push( CollectedDataFrame( collectedSignalsGroup ) ); } void diff --git a/src/AaosVhalSource.h b/src/AaosVhalSource.h index 998ac39c..85669672 100644 --- a/src/AaosVhalSource.h +++ b/src/AaosVhalSource.h @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace Aws @@ -25,9 +26,9 @@ class AaosVhalSource : public CustomDataSource { public: /** - * @param signalBufferPtr the signal buffer is used pass extracted data + * @param signalBufferDistributor Signal buffer distributor */ - AaosVhalSource( SignalBufferPtr signalBufferPtr ); + AaosVhalSource( SignalBufferDistributorPtr signalBufferDistributor ); /** * Initialize AaosVhalSource and set filter for CustomDataSource * @@ -55,7 +56,7 @@ class AaosVhalSource : public CustomDataSource * @param signalId Signal ID * @param value Property value */ - void setVehicleProperty( SignalID signalId, double value ); + void setVehicleProperty( SignalID signalId, const DecodedSignalValue &value ); static constexpr const char *CAN_CHANNEL_NUMBER = "canChannel"; static constexpr const char *CAN_RAW_FRAME_ID = "canFrameId"; @@ -65,7 +66,8 @@ class AaosVhalSource : public CustomDataSource const char *getThreadName() override; private: - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; + std::unordered_map mSignalIdToSignalType; std::shared_ptr mClock = ClockHandler::getClock(); CANChannelNumericID mCanChannel{ INVALID_CAN_SOURCE_NUMERIC_ID }; CANRawFrameID mCanRawFrameId{ 0 }; diff --git a/src/Assert.h b/src/Assert.h index d41ce02e..040dbeb3 100644 --- a/src/Assert.h +++ b/src/Assert.h @@ -8,6 +8,12 @@ #include #include +#ifdef __COVERITY__ +#include +#endif + +// coverity[autosar_cpp14_m16_3_2_violation] '#' operator required for debug text only +// coverity[misra_cpp_2008_rule_16_3_2_violation] '#' operator required for debug text only #define FWE_FATAL_ASSERT( cond, msg ) \ do \ { \ @@ -16,10 +22,12 @@ FWE_LOG_ERROR( "Fatal error condition occurred, aborting application: " + std::string( msg ) + " " + \ std::string( #cond ) ); \ LoggingModule::flush(); \ - abort(); \ + fatalError(); \ } \ - } while ( 0 ) + } while ( false ) +// coverity[autosar_cpp14_m16_3_2_violation] '#' operator required for debug text only +// coverity[misra_cpp_2008_rule_16_3_2_violation] '#' operator required for debug text only #define FWE_GRACEFUL_FATAL_ASSERT( cond, msg, returnValue ) \ do \ { \ @@ -31,4 +39,24 @@ std::raise( SIGUSR1 ); \ return returnValue; \ } \ - } while ( 0 ) + } while ( false ) + +namespace Aws +{ +namespace IoTFleetWise +{ + +inline void +fatalError() +{ +#ifdef __COVERITY__ + // coverity[misra_cpp_2008_rule_5_2_12_violation] Error from cassert header + // coverity[autosar_cpp14_m5_2_12_violation] Error from cassert header + assert( false ); +#else + abort(); +#endif +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsBootstrap.cpp b/src/AwsBootstrap.cpp index b9597b59..bc206d2e 100644 --- a/src/AwsBootstrap.cpp +++ b/src/AwsBootstrap.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 #include "AwsBootstrap.h" -#include "AwsSDKMemoryManager.h" #include "LoggingModule.h" #include #include @@ -27,11 +26,6 @@ struct AwsBootstrap::Impl // Enable for logging // mOptions.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info; - // We are using a memory manager that is not suitable for over-aligned types. - // Since we do not use alignas in our codebase, we are OK. - auto &memMgr = AwsSDKMemoryManager::getInstance(); - mOptions.memoryManagementOptions.memoryManager = &memMgr; - auto clientBootstrapFn = [this]() -> std::shared_ptr { // You need an event loop group to process IO events. // If you only have a few connections, 1 thread is ideal diff --git a/src/AwsGGChannel.cpp b/src/AwsGGChannel.cpp deleted file mode 100644 index 6cef70f2..00000000 --- a/src/AwsGGChannel.cpp +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "AwsGGChannel.h" -#include "AwsSDKMemoryManager.h" -#include "CacheAndPersist.h" -#include "IConnectivityModule.h" -#include "IReceiver.h" -#include "LoggingModule.h" -#include "TimeTypes.h" -#include -#include -#include -#include -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -// coverity[autosar_cpp14_a0_1_3_violation] false positive - function overrides sdk's virtual function. -void -SubscribeStreamHandler::OnStreamEvent( Aws::Greengrass::IoTCoreMessage *response ) -{ - auto message = response->GetMessage(); - - if ( message.has_value() && message.value().GetPayload().has_value() ) - { - Timestamp currentTime = mClock->monotonicTimeSinceEpochMs(); - - auto payloadBytes = message.value().GetPayload().value(); - std::string payloadString( payloadBytes.begin(), payloadBytes.end() ); - std::unordered_map properties; - auto messageUserProperties = message.value().GetUserProperties(); - if ( messageUserProperties.has_value() ) - { - for ( auto &property : messageUserProperties.value() ) - { - std::string name; - std::string value; - if ( property.GetKey().has_value() ) - { - auto propertyKey = property.GetKey().value(); - name = std::string( propertyKey.begin(), propertyKey.end() ); - } - if ( property.GetValue().has_value() ) - { - auto propertyValue = property.GetValue().value(); - value = std::string( propertyValue.begin(), propertyValue.end() ); - } - - if ( !properties.emplace( name, value ).second ) - { - FWE_LOG_WARN( "Duplicate property name '" + name + "', first value will be kept" ); - } - } - } - - auto messageExpiryIntervalSec = message.value().GetMessageExpiryIntervalSeconds(); - Timestamp messageExpiryMonotonicTimeSinceEpochMs = 0; - if ( messageExpiryIntervalSec.has_value() ) - { - // convert seconds to milliseconds and calculate absolute message expiry time - messageExpiryMonotonicTimeSinceEpochMs = - currentTime + static_cast( messageExpiryIntervalSec.value() ) * 1000; - } - mCallback( ReceivedChannelMessage{ - payloadBytes.data(), payloadBytes.size(), properties, messageExpiryMonotonicTimeSinceEpochMs } ); - } -} - -AwsGGChannel::AwsGGChannel( IConnectivityModule *connectivityModule, - std::shared_ptr payloadManager, - std::shared_ptr &ggConnection, - std::string topicName, - bool subscription ) - : mConnectivityModule( connectivityModule ) - , mPayloadManager( std::move( payloadManager ) ) - , mConnection( ggConnection ) - , mSubscribed( false ) - , mSubscription( subscription ) - , mTopicName( std::move( topicName ) ) -{ -} - -bool -AwsGGChannel::isAlive() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - return isAliveNotThreadSafe(); -} - -bool -AwsGGChannel::isAliveNotThreadSafe() -{ - if ( mConnectivityModule == nullptr ) - { - return false; - } - return mConnectivityModule->isAlive() && ( ( !mSubscription ) || mSubscribed ); -} - -ConnectivityError -AwsGGChannel::subscribe() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_ERROR( "Empty ingestion topic name provided" ); - return ConnectivityError::NotConfigured; - } - if ( !mConnectivityModule->isAlive() ) - { - FWE_LOG_ERROR( "MQTT Connection not established, failed to subscribe" ); - return ConnectivityError::NoConnection; - } - - mSubscribeStreamHandler = - std::make_shared( [&]( const ReceivedChannelMessage &receivedChannelMessage ) { - mListeners.notify( receivedChannelMessage ); - } ); - - if ( mConnection == nullptr ) - { - FWE_LOG_ERROR( "mConnection is null, not initialised" ) - return ConnectivityError::NoConnection; - } - mSubscribeOperation = mConnection->NewSubscribeToIoTCore( mSubscribeStreamHandler ); - Aws::Greengrass::SubscribeToIoTCoreRequest subscribeRequest; - subscribeRequest.SetQos( Aws::Greengrass::QOS_AT_LEAST_ONCE ); - subscribeRequest.SetTopicName( mTopicName.c_str() != nullptr ? mTopicName.c_str() : "" ); - - FWE_LOG_TRACE( "Attempting to subscribe to " + mTopicName + " topic" ); - auto requestStatus = mSubscribeOperation->Activate( subscribeRequest ).get(); - if ( !requestStatus ) - { - FWE_LOG_ERROR( "Failed to send subscription request to " + mTopicName + " topic" ); - return ConnectivityError::NoConnection; - } - - auto subscribeResultFuture = mSubscribeOperation->GetResult(); - - // To avoid throwing exceptions, wait on the result for a specified timeout: - if ( subscribeResultFuture.wait_for( std::chrono::seconds( 10 ) ) == std::future_status::timeout ) - { - FWE_LOG_ERROR( "Timed out while waiting for response from Greengrass Core" ); - return ConnectivityError::NoConnection; - } - - auto subscribeResult = subscribeResultFuture.get(); - if ( subscribeResult ) - { - FWE_LOG_TRACE( "Successfully subscribed to " + mTopicName + " topic" ); - mPayloadCountSent++; - } - else - { - auto errorType = subscribeResult.GetResultType(); - if ( errorType == OPERATION_ERROR ) - { - OperationError *error = subscribeResult.GetOperationError(); - /* - * This pointer can be casted to any error type like so: - * if(error->GetModelName() == UnauthorizedError::MODEL_NAME) - * UnauthorizedError *unauthorizedError = static_cast(error); - */ - if ( error->GetMessage().has_value() ) - { - auto errString = error->GetMessage().value().c_str(); - FWE_LOG_ERROR( "Greengrass Core responded with an error: " + - ( errString != nullptr ? std::string( errString ) : std::string( "Unknown error" ) ) ); - } - } - else - { - auto errString = subscribeResult.GetRpcError().StatusToString(); - FWE_LOG_ERROR( "Attempting to receive the response from the server failed with error code: " + - std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); - } - return ConnectivityError::NoConnection; - } - - return ConnectivityError::Success; -} - -size_t -AwsGGChannel::getMaxSendSize() const -{ - return AWS_IOT_MAX_MESSAGE_SIZE; -} - -ConnectivityError -AwsGGChannel::sendBuffer( const std::uint8_t *buf, size_t size, CollectionSchemeParams collectionSchemeParams ) -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_WARN( "Invalid topic provided" ); - return ConnectivityError::NotConfigured; - } - - if ( ( buf == nullptr ) || ( size == 0 ) ) - { - FWE_LOG_WARN( "No valid data provided" ); - return ConnectivityError::WrongInputData; - } - - if ( size > getMaxSendSize() ) - { - FWE_LOG_WARN( "Payload provided is too long" ); - return ConnectivityError::WrongInputData; - } - - if ( !isAliveNotThreadSafe() ) - { - FWE_LOG_WARN( "No alive IPC Connection." ); - if ( mPayloadManager != nullptr ) - { - if ( collectionSchemeParams.persist ) - { - mPayloadManager->storeData( buf, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - } - return ConnectivityError::NoConnection; - } - - if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) - { - FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + - " because IoT device SDK allocated the maximum defined memory. Payload will be stored" ); - - if ( collectionSchemeParams.persist ) - { - mPayloadManager->storeData( buf, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - return ConnectivityError::QuotaReached; - } - - if ( mConnection == nullptr ) - { - FWE_LOG_ERROR( "mConnection is null, not initialised" ) - return ConnectivityError::NoConnection; - } - auto publishOperation = mConnection->NewPublishToIoTCore(); - Aws::Greengrass::PublishToIoTCoreRequest publishRequest; - publishRequest.SetTopicName( mTopicName.c_str() != nullptr ? mTopicName.c_str() : "" ); - Aws::Crt::Vector payload( buf, buf + size ); - publishRequest.SetPayload( payload ); - publishRequest.SetQos( Aws::Greengrass::QOS_AT_LEAST_ONCE ); - - FWE_LOG_TRACE( "Attempting to publish to " + mTopicName + " topic" ); - auto requestStatus = publishOperation->Activate( publishRequest ).get(); - if ( !requestStatus ) - { - auto errString = requestStatus.StatusToString(); - FWE_LOG_ERROR( "Failed to publish to " + mTopicName + " topic with error " + - std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); - - return ConnectivityError::NoConnection; - } - - auto publishResultFuture = publishOperation->GetResult(); - - // To avoid throwing exceptions, wait on the result for a specified timeout: - if ( publishResultFuture.wait_for( std::chrono::seconds( 10 ) ) == std::future_status::timeout ) - { - FWE_LOG_ERROR( "Timed out while waiting for response from Greengrass Core" ); - return ConnectivityError::NoConnection; - } - - auto publishResult = publishResultFuture.get(); - if ( !publishResult ) - { - FWE_LOG_ERROR( "Failed to publish to " + mTopicName + " topic" ); - auto errorType = publishResult.GetResultType(); - if ( errorType == OPERATION_ERROR ) - { - OperationError *error = publishResult.GetOperationError(); - /* - * This pointer can be casted to any error type like so: - * if(error->GetModelName() == UnauthorizedError::MODEL_NAME) - * UnauthorizedError *unauthorizedError = static_cast(error); - */ - if ( error->GetMessage().has_value() ) - { - auto errString = error->GetMessage().value().c_str(); - FWE_LOG_ERROR( "Greengrass Core responded with an error: " + - ( errString != nullptr ? std::string( errString ) : std::string( "Unknown error" ) ) ); - } - } - else - { - auto errString = publishResult.GetRpcError().StatusToString(); - FWE_LOG_ERROR( "Attempting to receive the response from the server failed with error code " + - std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); - } - return ConnectivityError::NoConnection; - } - - return ConnectivityError::Success; -} - -ConnectivityError -AwsGGChannel::sendFile( const std::string &filePath, size_t size, CollectionSchemeParams collectionSchemeParams ) -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_WARN( "Invalid topic provided" ); - return ConnectivityError::NotConfigured; - } - - if ( mPayloadManager == nullptr ) - { - FWE_LOG_WARN( "No payload manager provided" ); - return ConnectivityError::NotConfigured; - } - - if ( filePath.empty() ) - { - FWE_LOG_WARN( "No valid file path provided" ); - return ConnectivityError::WrongInputData; - } - - if ( size > getMaxSendSize() ) - { - FWE_LOG_WARN( "Payload provided is too long" ); - return ConnectivityError::WrongInputData; - } - - if ( !isAliveNotThreadSafe() ) - { - if ( collectionSchemeParams.persist ) - { - // Only store metadata, file is already written on the disk - mPayloadManager->storeMetadata( filePath, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - return ConnectivityError::NoConnection; - } - - if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) - { - FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + - " because IoT device SDK allocated the maximum defined memory" ); - { - if ( collectionSchemeParams.persist ) - { - // Only store metadata, file is already written on the disk - mPayloadManager->storeMetadata( filePath, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - } - return ConnectivityError::QuotaReached; - } - - Aws::Crt::Vector payload( size ); - if ( mPayloadManager->retrievePayload( payload.data(), payload.size(), filePath ) != ErrorCode::SUCCESS ) - { - return ConnectivityError::WrongInputData; - } - - if ( mConnection == nullptr ) - { - FWE_LOG_ERROR( "mConnection is null, not initialised" ) - return ConnectivityError::NoConnection; - } - auto publishOperation = mConnection->NewPublishToIoTCore(); - Aws::Greengrass::PublishToIoTCoreRequest publishRequest; - publishRequest.SetTopicName( mTopicName.c_str() != nullptr ? mTopicName.c_str() : "" ); - publishRequest.SetPayload( payload ); - publishRequest.SetQos( Aws::Greengrass::QOS_AT_LEAST_ONCE ); - - FWE_LOG_TRACE( "Attempting to publish to " + mTopicName + " topic" ); - auto requestStatus = publishOperation->Activate( publishRequest ).get(); - if ( !requestStatus ) - { - FWE_LOG_ERROR( "Failed to publish to " + mTopicName + " topic with error " + - std::string( requestStatus.StatusToString().c_str() != nullptr - ? requestStatus.StatusToString().c_str() - : "" ) ); - return ConnectivityError::NoConnection; - } - - auto publishResultFuture = publishOperation->GetResult(); - - // To avoid throwing exceptions, wait on the result for a specified timeout: - if ( publishResultFuture.wait_for( std::chrono::seconds( 10 ) ) == std::future_status::timeout ) - { - FWE_LOG_ERROR( "Timed out while waiting for response from Greengrass Core" ); - return ConnectivityError::NoConnection; - } - - auto publishResult = publishResultFuture.get(); - if ( !publishResult ) - { - FWE_LOG_ERROR( "Failed to publish to " + mTopicName + " topic" ); - auto errorType = publishResult.GetResultType(); - if ( errorType == OPERATION_ERROR ) - { - OperationError *error = publishResult.GetOperationError(); - /* - * This pointer can be casted to any error type like so: - * if(error->GetModelName() == UnauthorizedError::MODEL_NAME) - * UnauthorizedError *unauthorizedError = static_cast(error); - */ - if ( error->GetMessage().has_value() ) - { - - auto errString = error->GetMessage().value().c_str(); - FWE_LOG_ERROR( "Greengrass Core responded with an error: " + - ( errString != nullptr ? std::string( errString ) : std::string( "Unknown error" ) ) ); - } - } - else - { - auto errString = publishResult.GetRpcError().StatusToString(); - FWE_LOG_ERROR( "Attempting to receive the response from the server failed with error code " + - std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); - } - return ConnectivityError::NoConnection; - } - - return ConnectivityError::Success; -} - -void -AwsGGChannel::subscribeToDataReceived( OnDataReceivedCallback callback ) -{ - mListeners.subscribe( callback ); -} - -bool -AwsGGChannel::unsubscribe() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( isAliveNotThreadSafe() ) - { - mSubscribeOperation->Close().wait(); - mSubscribed = false; - return true; - } - return false; -} - -AwsGGChannel::~AwsGGChannel() -{ - unsubscribe(); -} - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/AwsGGChannel.h b/src/AwsGGChannel.h deleted file mode 100644 index a55c5315..00000000 --- a/src/AwsGGChannel.h +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "Clock.h" -#include "ClockHandler.h" -#include "IConnectionTypes.h" -#include "IConnectivityChannel.h" -#include "IConnectivityModule.h" -#include "IReceiver.h" -#include "ISender.h" -#include "Listener.h" -#include "PayloadManager.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -using SubscribeCallback = std::function; - -class SubscribeStreamHandler : public Aws::Greengrass::SubscribeToIoTCoreStreamHandler -{ -public: - SubscribeStreamHandler( SubscribeCallback callback ) - : mCallback( std::move( callback ) ) - { - } - virtual ~SubscribeStreamHandler() = default; - -private: - // coverity[autosar_cpp14_a0_1_3_violation] false positive - function overrides sdk's virtual function. - void OnStreamEvent( Aws::Greengrass::IoTCoreMessage *response ) override; - - SubscribeCallback mCallback; - - /** - * @brief Clock member variable used to generate the time an MQTT message was received - */ - std::shared_ptr mClock = ClockHandler::getClock(); -}; - -/** - * @brief a channel that can be used as IReceiver or ISender or both - * - * If the Channel should be used for receiving data subscribe must be called. - * setTopic must be called always. There can be multiple AwsGGChannels - * from one AwsGGConnectivityModule. The channel of the connectivityModule passed in the - * constructor must be established before anything meaningful can be done with this class - * @see AwsGGConnectivityModule - */ -class AwsGGChannel : public IConnectivityChannel -{ -public: - AwsGGChannel( IConnectivityModule *connectivityModule, - std::shared_ptr payloadManager, - std::shared_ptr &ggConnection, - std::string topicName, - bool subscription ); - ~AwsGGChannel() override; - - AwsGGChannel( const AwsGGChannel & ) = delete; - AwsGGChannel &operator=( const AwsGGChannel & ) = delete; - AwsGGChannel( AwsGGChannel && ) = delete; - AwsGGChannel &operator=( AwsGGChannel && ) = delete; - - /** - * @brief Subscribe to the MQTT topic from setTopic. Necessary if data is received on the topic - * - * This function blocks until subscribe succeeded or failed and should be done in the setup form - * the bootstrap thread. The connection of the connectivityModule passed in the constructor - * must be established otherwise subscribe will fail. No retries are done to try to subscribe - * this needs to be done in the bootstrap during the setup. - * @return Success if subscribe finished correctly - */ - ConnectivityError subscribe(); - - /** - * @brief After unsubscribe no data will be received over the channel - * @return True for success - */ - bool unsubscribe(); - - bool isAlive() override; - - size_t getMaxSendSize() const override; - - ConnectivityError sendBuffer( const std::uint8_t *buf, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) override; - - ConnectivityError sendFile( const std::string &filePath, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) override; - - void subscribeToDataReceived( OnDataReceivedCallback callback ) override; - - void - invalidateConnection() - { - std::lock_guard connectivityLock( mConnectivityMutex ); - std::lock_guard connectivityLambdaLock( mConnectivityLambdaMutex ); - mConnectivityModule = nullptr; - } - - bool - shouldSubscribeAsynchronously() const - { - return mSubscription; - } - - /** - * @brief Returns the number of payloads successfully passed to the AWS IoT SDK - * @return Number of payloads - */ - unsigned - getPayloadCountSent() const override - { - return mPayloadCountSent; - } - -private: - bool isAliveNotThreadSafe(); - bool - isTopicValid() - { - return !mTopicName.empty(); - }; - /** See "Message size" : "The payload for every publish request can be no larger - * than 128 KB. AWS IoT Core rejects publish and connect requests larger than this size." - * https://docs.aws.amazon.com/general/latest/gr/iot-core.html#limits_iot - */ - static const size_t AWS_IOT_MAX_MESSAGE_SIZE = 131072; // = 128 KiB - IConnectivityModule *mConnectivityModule; - ThreadSafeListeners mListeners; - - std::mutex mConnectivityMutex; - std::mutex mConnectivityLambdaMutex; - std::shared_ptr mPayloadManager; - std::shared_ptr &mConnection; - std::atomic mSubscribed; - bool mSubscription; - std::atomic mPayloadCountSent{}; - - std::string mTopicName; - std::shared_ptr mSubscribeStreamHandler; - std::shared_ptr mSubscribeOperation; -}; - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/AwsGGConnectivityModule.cpp b/src/AwsGGConnectivityModule.cpp deleted file mode 100644 index 737db248..00000000 --- a/src/AwsGGConnectivityModule.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "AwsGGConnectivityModule.h" -#include "AwsGGChannel.h" -#include "LoggingModule.h" -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -AwsGGConnectivityModule::AwsGGConnectivityModule( Aws::Crt::Io::ClientBootstrap *clientBootstrap ) - : mConnected( false ) - , mConnectionEstablished( false ) - , mClientBootstrap( clientBootstrap ) -{ -} - -/* - * As first step to enable backend communication this code is mainly oriented on the basic_pub_sub - * example from the Aws Iot C++ SDK - */ -bool -AwsGGConnectivityModule::connect() -{ - - mConnected = false; - - mLifecycleHandler = std::make_unique(); - mConnection = std::make_shared( *( mClientBootstrap ) ); - auto connectionStatus = mConnection->Connect( *( mLifecycleHandler.get() ) ).get(); - if ( !connectionStatus ) - { - FWE_LOG_ERROR( "Failed to establish connection with error " + std::to_string( connectionStatus ) ); - return false; - } - - mConnected = true; - for ( auto channel : mChannels ) - { - if ( channel->shouldSubscribeAsynchronously() ) - { - channel->subscribe(); - } - } - FWE_LOG_INFO( "Successfully connected" ); - return true; -} - -std::shared_ptr -AwsGGConnectivityModule::createNewChannel( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription ) -{ - auto channel = std::make_shared( this, payloadManager, mConnection, topicName, subscription ); - mChannels.emplace_back( channel ); - return channel; -} - -bool -AwsGGConnectivityModule::disconnect() -{ - for ( auto channel : mChannels ) - { - channel->unsubscribe(); - channel->invalidateConnection(); - } - return true; -} - -AwsGGConnectivityModule::~AwsGGConnectivityModule() -{ - AwsGGConnectivityModule::disconnect(); -} - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/AwsGreengrassV2ConnectivityModule.cpp b/src/AwsGreengrassV2ConnectivityModule.cpp new file mode 100644 index 00000000..42483ff3 --- /dev/null +++ b/src/AwsGreengrassV2ConnectivityModule.cpp @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "AwsGreengrassV2ConnectivityModule.h" +#include "AwsGreengrassV2Receiver.h" +#include "AwsGreengrassV2Sender.h" +#include "LoggingModule.h" +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +AwsGreengrassV2ConnectivityModule::AwsGreengrassV2ConnectivityModule( Aws::Crt::Io::ClientBootstrap *clientBootstrap ) + : mConnected( false ) + , mConnectionEstablished( false ) + , mClientBootstrap( clientBootstrap ) +{ +} + +/* + * As first step to enable backend communication this code is mainly oriented on the basic_pub_sub + * example from the Aws Iot C++ SDK + */ +bool +AwsGreengrassV2ConnectivityModule::connect() +{ + + mConnected = false; + + mLifecycleHandler = std::make_unique(); + mGreengrassClient = std::make_shared( *( mClientBootstrap ) ); + auto connectionStatus = mGreengrassClient->Connect( *( mLifecycleHandler.get() ) ).get(); + if ( !connectionStatus ) + { + FWE_LOG_ERROR( "Failed to establish connection with error " + std::to_string( connectionStatus ) ); + return false; + } + + mConnected = true; + for ( auto receiver : mReceivers ) + { + receiver->subscribe(); + } + // subscribe to all topics first before notifying listeners for connection + mConnectionEstablishedListeners.notify(); + + FWE_LOG_INFO( "Successfully connected" ); + return true; +} + +std::shared_ptr +AwsGreengrassV2ConnectivityModule::createSender( const std::string &topicName, QoS publishQoS ) +{ + auto greengrassPublishQoS = Aws::Greengrass::QOS_AT_MOST_ONCE; + switch ( publishQoS ) + { + case QoS::AT_MOST_ONCE: + greengrassPublishQoS = Aws::Greengrass::QOS_AT_MOST_ONCE; + break; + case QoS::AT_LEAST_ONCE: + greengrassPublishQoS = Aws::Greengrass::QOS_AT_LEAST_ONCE; + break; + } + + auto sender = std::make_shared( this, mGreengrassClient, topicName, greengrassPublishQoS ); + mSenders.emplace_back( sender ); + return sender; +} + +std::shared_ptr +AwsGreengrassV2ConnectivityModule::createReceiver( const std::string &topicName ) +{ + auto receiver = std::make_shared( this, mGreengrassClient, topicName ); + mReceivers.emplace_back( receiver ); + return receiver; +} + +void +AwsGreengrassV2ConnectivityModule::subscribeToConnectionEstablished( OnConnectionEstablishedCallback callback ) +{ + mConnectionEstablishedListeners.subscribe( callback ); +} + +bool +AwsGreengrassV2ConnectivityModule::disconnect() +{ + for ( auto receiver : mReceivers ) + { + receiver->unsubscribe(); + receiver->invalidateConnection(); + } + + for ( auto sender : mSenders ) + { + sender->invalidateConnection(); + } + + return true; +} + +AwsGreengrassV2ConnectivityModule::~AwsGreengrassV2ConnectivityModule() +{ + AwsGreengrassV2ConnectivityModule::disconnect(); +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsGGConnectivityModule.h b/src/AwsGreengrassV2ConnectivityModule.h similarity index 51% rename from src/AwsGGConnectivityModule.h rename to src/AwsGreengrassV2ConnectivityModule.h index b56b065e..3e9e11ad 100644 --- a/src/AwsGGConnectivityModule.h +++ b/src/AwsGreengrassV2ConnectivityModule.h @@ -3,11 +3,13 @@ #pragma once -#include "AwsGGChannel.h" -#include "IConnectivityChannel.h" +#include "AwsGreengrassV2Receiver.h" +#include "AwsGreengrassV2Sender.h" #include "IConnectivityModule.h" +#include "IReceiver.h" +#include "ISender.h" +#include "Listener.h" #include "LoggingModule.h" -#include "PayloadManager.h" #include #include #include @@ -53,19 +55,19 @@ class IpcLifecycleHandler : public ConnectionLifecycleHandler }; /** - * @brief bootstrap of the Aws GG connectivity module. Only one object of this should normally exist + * @brief bootstrap of the AWS IoT Greengrass connectivity module. Only one object of this should normally exist * */ -class AwsGGConnectivityModule : public IConnectivityModule +class AwsGreengrassV2ConnectivityModule : public IConnectivityModule { public: - AwsGGConnectivityModule( Aws::Crt::Io::ClientBootstrap *clientBootstrap ); - ~AwsGGConnectivityModule() override; + AwsGreengrassV2ConnectivityModule( Aws::Crt::Io::ClientBootstrap *clientBootstrap ); + ~AwsGreengrassV2ConnectivityModule() override; - AwsGGConnectivityModule( const AwsGGConnectivityModule & ) = delete; - AwsGGConnectivityModule &operator=( const AwsGGConnectivityModule & ) = delete; - AwsGGConnectivityModule( AwsGGConnectivityModule && ) = delete; - AwsGGConnectivityModule &operator=( AwsGGConnectivityModule && ) = delete; + AwsGreengrassV2ConnectivityModule( const AwsGreengrassV2ConnectivityModule & ) = delete; + AwsGreengrassV2ConnectivityModule &operator=( const AwsGreengrassV2ConnectivityModule & ) = delete; + AwsGreengrassV2ConnectivityModule( AwsGreengrassV2ConnectivityModule && ) = delete; + AwsGreengrassV2ConnectivityModule &operator=( AwsGreengrassV2ConnectivityModule && ) = delete; bool connect() override; @@ -77,27 +79,20 @@ class AwsGGConnectivityModule : public IConnectivityModule return mConnected; }; - /** - * @brief create a new channel sharing the connection of this module - * This call needs to be done before calling connect for all asynchronous subscribe channel - * @param payloadManager the payload manager used by the new channel, - * @param topicName the topic which this channel should subscribe/publish to - * @param subscription whether the channel should subscribe to the topic. Otherwise it will - * just publish to it. - * - * @return a pointer to the newly created channel. A reference to the newly created channel is also hold inside this - * module. - */ - std::shared_ptr createNewChannel( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription = false ) override; + std::shared_ptr createSender( const std::string &topicName, QoS publishQoS = QoS::AT_MOST_ONCE ) override; + + std::shared_ptr createReceiver( const std::string &topicName ) override; + + void subscribeToConnectionEstablished( OnConnectionEstablishedCallback callback ) override; private: std::atomic mConnected; std::atomic mConnectionEstablished; - std::vector> mChannels; + ThreadSafeListeners mConnectionEstablishedListeners; + std::vector> mSenders; + std::vector> mReceivers; std::unique_ptr mLifecycleHandler; - std::shared_ptr mConnection; + std::shared_ptr mGreengrassClient; Aws::Crt::Io::ClientBootstrap *mClientBootstrap; }; diff --git a/src/AwsGreengrassV2Receiver.cpp b/src/AwsGreengrassV2Receiver.cpp new file mode 100644 index 00000000..d7e189c7 --- /dev/null +++ b/src/AwsGreengrassV2Receiver.cpp @@ -0,0 +1,171 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "AwsGreengrassV2Receiver.h" +#include "IConnectivityModule.h" +#include "IReceiver.h" +#include "LoggingModule.h" +#include "TimeTypes.h" +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +// coverity[autosar_cpp14_a0_1_3_violation] false positive - function overrides sdk's virtual function. +void +SubscribeStreamHandler::OnStreamEvent( Aws::Greengrass::IoTCoreMessage *response ) +{ + auto message = response->GetMessage(); + + if ( message.has_value() && message.value().GetPayload().has_value() && message.value().GetTopicName().has_value() ) + { + Timestamp currentTime = mClock->monotonicTimeSinceEpochMs(); + auto payloadBytes = message.value().GetPayload().value(); + auto mqttTopic = std::string( message.value().GetTopicName().value().c_str() ); + + mCallback( ReceivedConnectivityMessage{ payloadBytes.data(), payloadBytes.size(), currentTime, mqttTopic } ); + } +} + +AwsGreengrassV2Receiver::AwsGreengrassV2Receiver( + IConnectivityModule *connectivityModule, + std::shared_ptr &greengrassClient, + std::string topicName ) + : mConnectivityModule( connectivityModule ) + , mGreengrassClient( greengrassClient ) + , mSubscribed( false ) + , mTopicName( std::move( topicName ) ) +{ +} + +AwsGreengrassV2Receiver::~AwsGreengrassV2Receiver() +{ + unsubscribe(); +} + +bool +AwsGreengrassV2Receiver::isAlive() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + return isAliveNotThreadSafe(); +} + +bool +AwsGreengrassV2Receiver::isAliveNotThreadSafe() +{ + if ( mConnectivityModule == nullptr ) + { + return false; + } + return mConnectivityModule->isAlive() && mSubscribed; +} + +ConnectivityError +AwsGreengrassV2Receiver::subscribe() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + if ( mTopicName.empty() ) + { + FWE_LOG_ERROR( "Empty ingestion topic name provided" ); + return ConnectivityError::NotConfigured; + } + if ( !mConnectivityModule->isAlive() ) + { + FWE_LOG_ERROR( "MQTT Connection not established, failed to subscribe" ); + return ConnectivityError::NoConnection; + } + + mSubscribeStreamHandler = + std::make_shared( [&]( const ReceivedConnectivityMessage &receivedMessage ) { + mListeners.notify( receivedMessage ); + } ); + + if ( mGreengrassClient == nullptr ) + { + FWE_LOG_ERROR( "mGreengrassClient is null, not initialised" ) + return ConnectivityError::NoConnection; + } + mSubscribeOperation = mGreengrassClient->NewSubscribeToIoTCore( mSubscribeStreamHandler ); + Aws::Greengrass::SubscribeToIoTCoreRequest subscribeRequest; + subscribeRequest.SetQos( Aws::Greengrass::QOS_AT_LEAST_ONCE ); + subscribeRequest.SetTopicName( mTopicName.c_str() != nullptr ? mTopicName.c_str() : "" ); + + FWE_LOG_TRACE( "Attempting to subscribe to " + mTopicName + " topic" ); + auto requestStatus = mSubscribeOperation->Activate( subscribeRequest ).get(); + if ( !requestStatus ) + { + FWE_LOG_ERROR( "Failed to send subscription request to " + mTopicName + " topic" ); + return ConnectivityError::NoConnection; + } + + auto subscribeResultFuture = mSubscribeOperation->GetResult(); + + // To avoid throwing exceptions, wait on the result for a specified timeout: + if ( subscribeResultFuture.wait_for( std::chrono::seconds( 10 ) ) == std::future_status::timeout ) + { + FWE_LOG_ERROR( "Timed out while waiting for response from Greengrass Core" ); + return ConnectivityError::NoConnection; + } + + auto subscribeResult = subscribeResultFuture.get(); + if ( subscribeResult ) + { + FWE_LOG_TRACE( "Successfully subscribed to " + mTopicName + " topic" ); + } + else + { + auto errorType = subscribeResult.GetResultType(); + if ( errorType == OPERATION_ERROR ) + { + OperationError *error = subscribeResult.GetOperationError(); + /* + * This pointer can be casted to any error type like so: + * if(error->GetModelName() == UnauthorizedError::MODEL_NAME) + * UnauthorizedError *unauthorizedError = static_cast(error); + */ + if ( error->GetMessage().has_value() ) + { + auto errString = error->GetMessage().value().c_str(); + FWE_LOG_ERROR( "Greengrass Core responded with an error: " + + ( errString != nullptr ? std::string( errString ) : std::string( "Unknown error" ) ) ); + } + } + else + { + auto errString = subscribeResult.GetRpcError().StatusToString(); + FWE_LOG_ERROR( "Attempting to receive the response from the server failed with error code: " + + std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); + } + return ConnectivityError::NoConnection; + } + + return ConnectivityError::Success; +} + +void +AwsGreengrassV2Receiver::subscribeToDataReceived( OnDataReceivedCallback callback ) +{ + mListeners.subscribe( callback ); +} + +bool +AwsGreengrassV2Receiver::unsubscribe() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + if ( isAliveNotThreadSafe() ) + { + mSubscribeOperation->Close().wait(); + mSubscribed = false; + return true; + } + return false; +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsGreengrassV2Receiver.h b/src/AwsGreengrassV2Receiver.h new file mode 100644 index 00000000..66c0bd8e --- /dev/null +++ b/src/AwsGreengrassV2Receiver.h @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "Clock.h" +#include "ClockHandler.h" +#include "IConnectionTypes.h" +#include "IConnectivityModule.h" +#include "IReceiver.h" +#include "Listener.h" +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +using SubscribeCallback = std::function; + +class SubscribeStreamHandler : public Aws::Greengrass::SubscribeToIoTCoreStreamHandler +{ +public: + SubscribeStreamHandler( SubscribeCallback callback ) + : mCallback( std::move( callback ) ) + { + } + virtual ~SubscribeStreamHandler() = default; + +private: + // coverity[autosar_cpp14_a0_1_3_violation] false positive - function overrides sdk's virtual function. + void OnStreamEvent( Aws::Greengrass::IoTCoreMessage *response ) override; + + SubscribeCallback mCallback; + + /** + * @brief Clock member variable used to generate the time an MQTT message was received + */ + std::shared_ptr mClock = ClockHandler::getClock(); +}; + +/** + * @brief A receiver to receive messages using AWS IoT Greengrass + * + * There can be multiple AwsGreengrassV2Receiver from one AwsGreengrassV2ConnectivityModule. The connection of the + * connectivityModule passed in the constructor must be established before anything meaningful + * can be done with this class. + * @see AwsGreengrassV2ConnectivityModule + */ +class AwsGreengrassV2Receiver : public IReceiver +{ +public: + AwsGreengrassV2Receiver( IConnectivityModule *connectivityModule, + std::shared_ptr &greengrassClient, + std::string topicName ); + ~AwsGreengrassV2Receiver() override; + + AwsGreengrassV2Receiver( const AwsGreengrassV2Receiver & ) = delete; + AwsGreengrassV2Receiver &operator=( const AwsGreengrassV2Receiver & ) = delete; + AwsGreengrassV2Receiver( AwsGreengrassV2Receiver && ) = delete; + AwsGreengrassV2Receiver &operator=( AwsGreengrassV2Receiver && ) = delete; + + /** + * @brief Subscribe to the MQTT topic from setTopic. Necessary if data is received on the topic + * + * This function blocks until subscribe succeeded or failed and should be done in the setup form + * the bootstrap thread. The connection of the connectivityModule passed in the constructor + * must be established otherwise subscribe will fail. No retries are done to try to subscribe + * this needs to be done in the bootstrap during the setup. + * @return Success if subscribe finished correctly + */ + ConnectivityError subscribe(); + + /** + * @brief After unsubscribe no data will be received by the receiver + * @return True for success + */ + bool unsubscribe(); + + bool isAlive() override; + + void subscribeToDataReceived( OnDataReceivedCallback callback ) override; + + void + invalidateConnection() + { + std::lock_guard connectivityLock( mConnectivityMutex ); + mConnectivityModule = nullptr; + } + +private: + bool isAliveNotThreadSafe(); + + IConnectivityModule *mConnectivityModule; + ThreadSafeListeners mListeners; + + std::mutex mConnectivityMutex; + std::shared_ptr &mGreengrassClient; + std::atomic mSubscribed; + + std::string mTopicName; + std::shared_ptr mSubscribeStreamHandler; + std::shared_ptr mSubscribeOperation; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsGreengrassV2Sender.cpp b/src/AwsGreengrassV2Sender.cpp new file mode 100644 index 00000000..97e5a1da --- /dev/null +++ b/src/AwsGreengrassV2Sender.cpp @@ -0,0 +1,185 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "AwsGreengrassV2Sender.h" +#include "AwsSDKMemoryManager.h" +#include "IConnectionTypes.h" +#include "IConnectivityModule.h" +#include "LoggingModule.h" +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +AwsGreengrassV2Sender::AwsGreengrassV2Sender( + IConnectivityModule *connectivityModule, + std::shared_ptr &greengrassClient, + std::string topicName, + Aws::Greengrass::QOS publishQoS ) + : mConnectivityModule( connectivityModule ) + , mGreengrassClient( greengrassClient ) + , mPublishQoS( publishQoS ) + , mTopicName( std::move( topicName ) ) +{ +} + +bool +AwsGreengrassV2Sender::isAlive() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + return isAliveNotThreadSafe(); +} + +bool +AwsGreengrassV2Sender::isAliveNotThreadSafe() +{ + if ( mConnectivityModule == nullptr ) + { + return false; + } + return mConnectivityModule->isAlive(); +} + +size_t +AwsGreengrassV2Sender::getMaxSendSize() const +{ + return AWS_IOT_MAX_MESSAGE_SIZE; +} + +void +AwsGreengrassV2Sender::sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) +{ + sendBufferToTopic( mTopicName, buf, size, callback ); +} + +void +AwsGreengrassV2Sender::sendBufferToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ) +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + if ( topic.empty() ) + { + FWE_LOG_WARN( "Invalid topic provided" ); + callback( ConnectivityError::NotConfigured ); + return; + } + + if ( ( buf == nullptr ) || ( size == 0 ) ) + { + FWE_LOG_WARN( "No valid data provided" ); + callback( ConnectivityError::WrongInputData ); + return; + } + + if ( size > getMaxSendSize() ) + { + FWE_LOG_WARN( "Payload provided is too long" ); + callback( ConnectivityError::WrongInputData ); + return; + } + + if ( !isAliveNotThreadSafe() ) + { + FWE_LOG_WARN( "No alive IPC Connection." ); + callback( ConnectivityError::NoConnection ); + return; + } + + if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) + { + FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + + " because IoT device SDK allocated the maximum defined memory." ); + + callback( ConnectivityError::QuotaReached ); + return; + } + + if ( mGreengrassClient == nullptr ) + { + FWE_LOG_ERROR( "mGreengrassClient is null, not initialised" ) + callback( ConnectivityError::NoConnection ); + return; + } + auto publishOperation = mGreengrassClient->NewPublishToIoTCore(); + Aws::Greengrass::PublishToIoTCoreRequest publishRequest; + publishRequest.SetTopicName( topic.c_str() != nullptr ? topic.c_str() : "" ); + Aws::Crt::Vector payload( buf, buf + size ); + publishRequest.SetPayload( payload ); + publishRequest.SetQos( mPublishQoS ); + + FWE_LOG_TRACE( "Attempting to publish to " + topic + " topic" ); + auto onMessageFlushCallback = [callback, topicName = topic]( int errorCode ) { + if ( errorCode != 0 ) + { + FWE_LOG_ERROR( "Failed to publish to " + topicName + " topic with error code " + + std::to_string( errorCode ) ); + callback( ConnectivityError::TransmissionError ); + return; + } + callback( ConnectivityError::Success ); + }; + auto requestStatus = publishOperation->Activate( publishRequest, onMessageFlushCallback ).get(); + if ( !requestStatus ) + { + auto errString = requestStatus.StatusToString(); + FWE_LOG_ERROR( "Failed to publish to " + topic + " topic with error " + + std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); + + callback( ConnectivityError::NoConnection ); + return; + } + + auto publishResultFuture = publishOperation->GetResult(); + + // To avoid throwing exceptions, wait on the result for a specified timeout: + if ( publishResultFuture.wait_for( std::chrono::seconds( 10 ) ) == std::future_status::timeout ) + { + FWE_LOG_ERROR( "Timed out while waiting for response from Greengrass Core" ); + callback( ConnectivityError::NoConnection ); + return; + } + + auto publishResult = publishResultFuture.get(); + if ( !publishResult ) + { + FWE_LOG_ERROR( "Failed to publish to " + topic + " topic" ); + auto errorType = publishResult.GetResultType(); + if ( errorType == OPERATION_ERROR ) + { + OperationError *error = publishResult.GetOperationError(); + /* + * This pointer can be casted to any error type like so: + * if(error->GetModelName() == UnauthorizedError::MODEL_NAME) + * UnauthorizedError *unauthorizedError = static_cast(error); + */ + if ( error->GetMessage().has_value() ) + { + auto errString = error->GetMessage().value().c_str(); + FWE_LOG_ERROR( "Greengrass Core responded with an error: " + + ( errString != nullptr ? std::string( errString ) : std::string( "Unknown error" ) ) ); + } + } + else + { + auto errString = publishResult.GetRpcError().StatusToString(); + FWE_LOG_ERROR( "Attempting to receive the response from the server failed with error code " + + std::string( errString.c_str() != nullptr ? errString.c_str() : "Unknown error" ) ); + } + callback( ConnectivityError::NoConnection ); + return; + } + + callback( ConnectivityError::Success ); +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsGreengrassV2Sender.h b/src/AwsGreengrassV2Sender.h new file mode 100644 index 00000000..6b12069a --- /dev/null +++ b/src/AwsGreengrassV2Sender.h @@ -0,0 +1,94 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "IConnectivityModule.h" +#include "ISender.h" +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +/** + * @brief A sender to send messages using AWS IoT Greengrass + * + * There can be multiple AwsGreengrassV2Sender from one AwsGreengrassV2ConnectivityModule. The connection of the + * connectivityModule passed in the constructor must be established before anything meaningful + * can be done with this class. + * @see AwsGreengrassV2ConnectivityModule + */ +class AwsGreengrassV2Sender : public ISender +{ +public: + AwsGreengrassV2Sender( IConnectivityModule *connectivityModule, + std::shared_ptr &greengrassClient, + std::string topicName, + Aws::Greengrass::QOS publishQoS ); + ~AwsGreengrassV2Sender() override = default; + + AwsGreengrassV2Sender( const AwsGreengrassV2Sender & ) = delete; + AwsGreengrassV2Sender &operator=( const AwsGreengrassV2Sender & ) = delete; + AwsGreengrassV2Sender( AwsGreengrassV2Sender && ) = delete; + AwsGreengrassV2Sender &operator=( AwsGreengrassV2Sender && ) = delete; + + bool isAlive() override; + + size_t getMaxSendSize() const override; + + void sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) override; + + void sendBufferToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ) override; + + void + invalidateConnection() + { + std::lock_guard connectivityLock( mConnectivityMutex ); + mConnectivityModule = nullptr; + } + + /** + * @brief Returns the number of payloads successfully passed to the AWS IoT SDK + * @return Number of payloads + */ + unsigned + getPayloadCountSent() const override + { + return mPayloadCountSent; + } + +private: + bool isAliveNotThreadSafe(); + bool + isTopicValid() + { + return !mTopicName.empty(); + }; + /** See "Message size" : "The payload for every publish request can be no larger + * than 128 KB. AWS IoT Core rejects publish and connect requests larger than this size." + * https://docs.aws.amazon.com/general/latest/gr/iot-core.html#limits_iot + */ + static const size_t AWS_IOT_MAX_MESSAGE_SIZE = 131072; // = 128 KiB + IConnectivityModule *mConnectivityModule; + + std::mutex mConnectivityMutex; + std::shared_ptr &mGreengrassClient; + Aws::Greengrass::QOS mPublishQoS; + std::atomic mPayloadCountSent{}; + + std::string mTopicName; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsIotChannel.cpp b/src/AwsIotChannel.cpp deleted file mode 100644 index 70398351..00000000 --- a/src/AwsIotChannel.cpp +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#include "AwsIotChannel.h" -#include "AwsSDKMemoryManager.h" -#include "CacheAndPersist.h" -#include "IConnectivityModule.h" -#include "LoggingModule.h" -#include "TimeTypes.h" -#include "TraceModule.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -AwsIotChannel::AwsIotChannel( IConnectivityModule *connectivityModule, - std::shared_ptr payloadManager, - std::shared_ptr &mqttClient, - std::string topicName, - bool subscription ) - : mConnectivityModule( connectivityModule ) - , mPayloadManager( std::move( payloadManager ) ) - , mMqttClient( mqttClient ) - , mTopicName( std::move( topicName ) ) - , mSubscribed( false ) - , mSubscription( subscription ) -{ -} - -bool -AwsIotChannel::isAlive() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - return isAliveNotThreadSafe(); -} - -bool -AwsIotChannel::isAliveNotThreadSafe() -{ - if ( mConnectivityModule == nullptr ) - { - return false; - } - return mConnectivityModule->isAlive() && ( ( !mSubscription ) || mSubscribed ); -} - -ConnectivityError -AwsIotChannel::subscribe() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_ERROR( "Empty ingestion topic name provided" ); - return ConnectivityError::NotConfigured; - } - if ( !mConnectivityModule->isAlive() ) - { - FWE_LOG_ERROR( "MQTT Connection not established, failed to subscribe" ); - return ConnectivityError::NoConnection; - } - - /* - * Subscribe for incoming publish messages on topic. - */ - std::promise subscribeFinishedPromise; - auto onSubAck = [&]( int errorCode, std::shared_ptr subAckPacket ) { - mSubscribed = false; - if ( errorCode != 0 ) - { - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); - auto errorString = Aws::Crt::ErrorDebugString( errorCode ); - FWE_LOG_ERROR( "Subscribe failed with error code " + std::to_string( errorCode ) + ": " + - std::string( errorString != nullptr ? errorString : "Unknown error" ) ); - subscribeFinishedPromise.set_value( false ); - return; - } - - std::string grantedQoS = "unknown"; - if ( subAckPacket != nullptr ) - { - for ( auto reasonCode : subAckPacket->getReasonCodes() ) - { - if ( reasonCode <= Aws::Crt::Mqtt5::SubAckReasonCode::AWS_MQTT5_SARC_GRANTED_QOS_2 ) - { - grantedQoS = std::to_string( reasonCode ); - } - else if ( reasonCode >= Aws::Crt::Mqtt5::SubAckReasonCode::AWS_MQTT5_SARC_UNSPECIFIED_ERROR ) - { - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); - // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null - auto reasonString = std::string( subAckPacket->getReasonString().has_value() - ? subAckPacket->getReasonString()->c_str() - : "Unknown reason" ); - FWE_LOG_ERROR( "Server rejected subscription to topic " + mTopicName + ". Reason code " + - std::to_string( reasonCode ) + ": " + reasonString ); - subscribeFinishedPromise.set_value( false ); - // Just return on the first error found. There could be multiple reason codes if we request multiple - // subscriptions at once, but we always request a single one. - return; - } - } - } - - FWE_LOG_TRACE( "Subscribe succeeded for topic " + mTopicName + " with QoS " + grantedQoS ); - mSubscribed = true; - subscribeFinishedPromise.set_value( true ); - }; - - FWE_LOG_TRACE( "Subscribing to topic " + mTopicName ); - // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null - Aws::Crt::Mqtt5::Subscription sub1( mTopicName.c_str(), Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_LEAST_ONCE ); - auto subPacket = std::make_shared(); - subPacket->WithSubscription( std::move( sub1 ) ); - - if ( !mMqttClient->Subscribe( subPacket, onSubAck ) ) - { - FWE_LOG_ERROR( "Subscribe failed" ); - return ConnectivityError::NoConnection; - } - - // Blocked call until subscribe finished this call should quickly either fail or succeed but - // depends on the network quality the Bootstrap needs to retry subscribing if failed. - subscribeFinishedPromise.get_future().wait(); - - if ( !mSubscribed ) - { - return ConnectivityError::NoConnection; - } - - return ConnectivityError::Success; -} - -size_t -AwsIotChannel::getMaxSendSize() const -{ - return AWS_IOT_MAX_MESSAGE_SIZE; -} - -ConnectivityError -AwsIotChannel::sendBuffer( const std::uint8_t *buf, size_t size, CollectionSchemeParams collectionSchemeParams ) -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_WARN( "Invalid topic provided" ); - return ConnectivityError::NotConfigured; - } - - if ( ( buf == nullptr ) || ( size == 0 ) ) - { - FWE_LOG_WARN( "No valid data provided" ); - return ConnectivityError::WrongInputData; - } - - if ( size > getMaxSendSize() ) - { - FWE_LOG_WARN( "Payload provided is too long" ); - return ConnectivityError::WrongInputData; - } - - if ( !isAliveNotThreadSafe() ) - { - if ( mPayloadManager != nullptr ) - { - if ( collectionSchemeParams.persist ) - { - mPayloadManager->storeData( buf, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - } - return ConnectivityError::NoConnection; - } - - if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) - { - FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + - " because IoT device SDK allocated the maximum defined memory. Payload will be stored" ); - - if ( collectionSchemeParams.persist ) - { - mPayloadManager->storeData( buf, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - return ConnectivityError::QuotaReached; - } - - publishMessage( buf, size ); - - return ConnectivityError::Success; -} - -ConnectivityError -AwsIotChannel::sendFile( const std::string &filePath, size_t size, CollectionSchemeParams collectionSchemeParams ) -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - if ( !isTopicValid() ) - { - FWE_LOG_WARN( "Invalid topic provided" ); - return ConnectivityError::NotConfigured; - } - - if ( mPayloadManager == nullptr ) - { - FWE_LOG_WARN( "No payload manager provided" ); - return ConnectivityError::NotConfigured; - } - - if ( filePath.empty() ) - { - FWE_LOG_WARN( "No valid file path provided" ); - return ConnectivityError::WrongInputData; - } - - if ( size > getMaxSendSize() ) - { - FWE_LOG_WARN( "Payload provided is too long" ); - return ConnectivityError::WrongInputData; - } - - if ( !isAliveNotThreadSafe() ) - { - if ( collectionSchemeParams.persist ) - { - // Only store metadata, file is already written on the disk - mPayloadManager->storeMetadata( filePath, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - return ConnectivityError::NoConnection; - } - - if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) - { - FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + - " because IoT device SDK allocated the maximum defined memory" ); - { - if ( collectionSchemeParams.persist ) - { - // Only store metadata, file is already written on the disk - mPayloadManager->storeMetadata( filePath, size, collectionSchemeParams ); - } - else - { - FWE_LOG_TRACE( "CollectionScheme does not activate persistency on disk" ); - } - } - return ConnectivityError::QuotaReached; - } - - std::vector payload( size ); - if ( mPayloadManager->retrievePayload( payload.data(), payload.size(), filePath ) != ErrorCode::SUCCESS ) - { - return ConnectivityError::WrongInputData; - } - - publishMessage( payload.data(), payload.size() ); - - return ConnectivityError::Success; -} - -void -AwsIotChannel::subscribeToDataReceived( OnDataReceivedCallback callback ) -{ - mListeners.subscribe( callback ); -} - -void -AwsIotChannel::onDataReceived( const Aws::Crt::Mqtt5::PublishReceivedEventData &eventData ) -{ - Timestamp currentTime = mClock->monotonicTimeSinceEpochMs(); - - std::unordered_map properties; - - auto correlationData = eventData.publishPacket->getCorrelationData(); - if ( correlationData.has_value() ) - { - // coverity[cert_str51_cpp_violation] correlationData->ptr is not null, checked before - properties[PROPERTY_NAME_CORRELATION_DATA] = - std::string( correlationData->ptr, correlationData->ptr + correlationData->len ); - } - - auto messageExpiryIntervalSec = eventData.publishPacket->getMessageExpiryIntervalSec(); - Timestamp messageExpiryMonotonicTimeSinceEpochMs = 0; - if ( messageExpiryIntervalSec.has_value() ) - { - // convert seconds to milliseconds and calculate absolute message expiry time - messageExpiryMonotonicTimeSinceEpochMs = currentTime + messageExpiryIntervalSec.value() * 1000; - } - - for ( auto property : eventData.publishPacket->getUserProperties() ) - { - auto name = std::string( property.getName().begin(), property.getName().end() ); - auto value = std::string( property.getValue().begin(), property.getValue().end() ); - if ( !properties.emplace( name, value ).second ) - { - FWE_LOG_WARN( "Duplicate property name '" + name + "', first value will be kept" ); - } - properties[name] = value; - } - mListeners.notify( ReceivedChannelMessage{ eventData.publishPacket->getPayload().ptr, - eventData.publishPacket->getPayload().len, - properties, - messageExpiryMonotonicTimeSinceEpochMs } ); -} - -void -AwsIotChannel::publishMessage( const uint8_t *buf, size_t size ) -{ - auto payload = Aws::Crt::ByteBufFromArray( buf, size ); - - auto onPublishComplete = [size, this]( int errorCode, - std::shared_ptr result ) mutable { - { - std::lock_guard connectivityLambdaLock( mConnectivityLambdaMutex ); - AwsSDKMemoryManager::getInstance().releaseReservedMemory( size ); - } - - if ( result->wasSuccessful() ) - { - FWE_LOG_TRACE( "Publish succeeded" ); - mPayloadCountSent++; - } - else - { - auto errorString = Aws::Crt::ErrorDebugString( errorCode ); - FWE_LOG_ERROR( std::string( "Operation failed with error" ) + - ( errorString != nullptr ? std::string( errorString ) : std::string( "Unknown error" ) ) ); - } - }; - std::shared_ptr publishPacket = - std::make_shared( mTopicName.c_str(), - Aws::Crt::ByteCursorFromByteBuf( payload ), - Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); - mMqttClient->Publish( publishPacket, onPublishComplete ); -} - -bool -AwsIotChannel::unsubscribe() -{ - auto result = unsubscribeAsync(); - result.wait(); - return result.get(); -} - -std::future -AwsIotChannel::unsubscribeAsync() -{ - std::lock_guard connectivityLock( mConnectivityMutex ); - - // We can't move the promise into the lambda, because the lambda needs to be copyable. So we - // don't have much choice but use a shared pointer. - auto unsubscribeFinishedPromise = std::make_shared>(); - auto unsubscribeFuture = unsubscribeFinishedPromise->get_future(); - - if ( !isAliveNotThreadSafe() ) - { - unsubscribeFinishedPromise->set_value( false ); - return unsubscribeFuture; - } - - FWE_LOG_TRACE( "Unsubscribing..." ); - auto unsubPacket = std::make_shared(); - // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null - unsubPacket->WithTopicFilter( mTopicName.c_str() ); - mMqttClient->Unsubscribe( - unsubPacket, - [this, unsubscribeFinishedPromise]( int errorCode, std::shared_ptr unsubAckPacket ) { - if ( errorCode != 0 ) - { - auto errorString = Aws::Crt::ErrorDebugString( errorCode ); - std::string logMessage = "Unsubscribe failed with error code " + std::to_string( errorCode ) + ": " + - std::string( errorString != nullptr ? errorString : "Unknown error" ); - if ( errorCode == AWS_ERROR_MQTT5_USER_REQUESTED_STOP ) - { - FWE_LOG_TRACE( logMessage ); - } - else - { - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); - FWE_LOG_ERROR( logMessage ); - } - unsubscribeFinishedPromise->set_value( false ); - return; - } - - if ( unsubAckPacket != nullptr ) - { - for ( Aws::Crt::Mqtt5::UnSubAckReasonCode reasonCode : unsubAckPacket->getReasonCodes() ) - { - if ( reasonCode >= Aws::Crt::Mqtt5::UnSubAckReasonCode::AWS_MQTT5_UARC_UNSPECIFIED_ERROR ) - { - // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null - auto reasonString = std::string( unsubAckPacket->getReasonString().has_value() - ? unsubAckPacket->getReasonString()->c_str() - : "Unknown reason" ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); - FWE_LOG_ERROR( "Server rejected unsubscribe from topic " + mTopicName + ". Reason code " + - std::to_string( reasonCode ) + ": " + reasonString ); - unsubscribeFinishedPromise->set_value( false ); - // Just return on the first error found. There could be multiple reason codes if we - // unsubscribe from multiple subscriptions at once, but we always request a single one. - return; - } - } - } - FWE_LOG_TRACE( "Unsubscribed from topic " + mTopicName ); - mSubscribed = false; - unsubscribeFinishedPromise->set_value( true ); - } ); - - return unsubscribeFuture; -} - -AwsIotChannel::~AwsIotChannel() -{ - unsubscribe(); -} - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/AwsIotChannel.h b/src/AwsIotChannel.h deleted file mode 100644 index fcd0322e..00000000 --- a/src/AwsIotChannel.h +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "Clock.h" -#include "ClockHandler.h" -#include "IConnectionTypes.h" -#include "IConnectivityChannel.h" -#include "IConnectivityModule.h" -#include "IReceiver.h" -#include "ISender.h" -#include "Listener.h" -#include "MqttClientWrapper.h" -#include "PayloadManager.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -/** - * @brief a channel that can be used as IReceiver or ISender or both - * - * If the Channel should be used for receiving data subscribe must be called. - * setTopic must be called always. There can be multiple AwsIotChannels - * from one AwsIotConnectivityModule. The channel of the connectivityModule passed in the - * constructor must be established before anything meaningful can be done with this class - * @see AwsIotConnectivityModule - */ -// coverity[cert_dcl60_cpp_violation] false positive - class only defined once -// coverity[autosar_cpp14_m3_2_2_violation] false positive - class only defined once -// coverity[misra_cpp_2008_rule_3_2_2_violation] false positive - class only defined once -class AwsIotChannel : public IConnectivityChannel -{ -public: - AwsIotChannel( IConnectivityModule *connectivityModule, - std::shared_ptr payloadManager, - std::shared_ptr &mqttClient, - std::string topicName, - bool subscription ); - ~AwsIotChannel() override; - - AwsIotChannel( const AwsIotChannel & ) = delete; - AwsIotChannel &operator=( const AwsIotChannel & ) = delete; - AwsIotChannel( AwsIotChannel && ) = delete; - AwsIotChannel &operator=( AwsIotChannel && ) = delete; - - /** - * @brief Subscribe to the MQTT topic from setTopic. Necessary if data is received on the topic - * - * This function blocks until subscribe succeeded or failed and should be done in the setup form - * the bootstrap thread. The connection of the connectivityModule passed in the constructor - * must be established otherwise subscribe will fail. No retries are done to try to subscribe - * this needs to be done in the bootstrap during the setup. - * @return Success if subscribe finished correctly - */ - ConnectivityError subscribe(); - - /** - * @brief After unsubscribe no data will be received over the channel - * @return True for success - */ - bool unsubscribe(); - - /** - * @brief Unsubscribe from the MQTT topic asynchronously - * @return A future that can be used to wait for the unsubscribe to finish. It will return True on success. - */ - std::future unsubscribeAsync(); - - bool isAlive() override; - - size_t getMaxSendSize() const override; - - ConnectivityError sendBuffer( const std::uint8_t *buf, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) override; - - ConnectivityError sendFile( const std::string &filePath, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) override; - - void subscribeToDataReceived( OnDataReceivedCallback callback ) override; - - void onDataReceived( const Aws::Crt::Mqtt5::PublishReceivedEventData &eventData ); - - void - invalidateConnection() - { - std::lock_guard connectivityLock( mConnectivityMutex ); - std::lock_guard connectivityLambdaLock( mConnectivityLambdaMutex ); - mConnectivityModule = nullptr; - }; - - bool - shouldSubscribeAsynchronously() const - { - return mSubscription; - }; - - /** - * @brief Returns the number of payloads successfully passed to the AWS IoT SDK - * @return Number of payloads - */ - unsigned - getPayloadCountSent() const override - { - return mPayloadCountSent; - } - -private: - bool isAliveNotThreadSafe(); - - // coverity[autosar_cpp14_a0_1_3_violation] false positive - function is used - bool - isTopicValid() - { - return !mTopicName.empty(); - }; - - void publishMessage( const uint8_t *buf, size_t size ); - - /** See "Message size" : "The payload for every publish request can be no larger - * than 128 KB. AWS IoT Core rejects publish and connect requests larger than this size." - * https://docs.aws.amazon.com/general/latest/gr/iot-core.html#limits_iot - */ - static const size_t AWS_IOT_MAX_MESSAGE_SIZE = 131072; // = 128 KiB - IConnectivityModule *mConnectivityModule; - ThreadSafeListeners mListeners; - std::shared_ptr mPayloadManager; - std::shared_ptr &mMqttClient; - std::mutex mConnectivityMutex; - std::mutex mConnectivityLambdaMutex; - std::string mTopicName; - std::atomic mSubscribed; - std::atomic mPayloadCountSent{}; - - /** - * @brief Clock member variable used to generate the time an MQTT message was received - */ - std::shared_ptr mClock = ClockHandler::getClock(); - - bool mSubscription; -}; - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/AwsIotConnectivityModule.cpp b/src/AwsIotConnectivityModule.cpp index bd644063..c59b09b9 100644 --- a/src/AwsIotConnectivityModule.cpp +++ b/src/AwsIotConnectivityModule.cpp @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include "AwsIotConnectivityModule.h" +#include "IConnectionTypes.h" #include "LoggingModule.h" #include "Thread.h" #include "TraceModule.h" -#include #include #include #include @@ -21,31 +21,26 @@ namespace Aws namespace IoTFleetWise { -// Default MQTT Keep alive in seconds. -// Defines how often an MQTT PING message is sent to the MQTT broker to keep the connection alive -// Default set to 60 seconds. Every 60 seconds the stack will send an MQTT PING req. -// The longer this interval is, the more the stack takes to detect the state of the TCP connection -// at the lower network layers. -// This parameter is asserted in the C SDK. It shall be strictly bigger than the default -// connection ping timeout( set to 3 seconds). -// Refer to https://github.com/awslabs/aws-c-mqtt/blob/a2ee9a321fcafa19b0473b88a54e0ae8dde5fddf/source/client.c#L1461 -constexpr uint16_t MQTT_CONNECT_KEEP_ALIVE_SECONDS = 60; -// Default ping timeout value in milliseconds -// If a response is not received within this interval, the connection will be reestablished. -// If the PING request does not return within this interval, the stack will create a new one. -constexpr uint32_t MQTT_PING_TIMEOUT_MS = 3000; -constexpr uint32_t MQTT_SESSION_EXPIRY_INTERVAL_SEC = 3600; - -// How much time to wait for a response to an unsubscribe operation when shutting the module down. -constexpr uint32_t MQTT_UNSUBSCRIBE_TIMEOUT_ON_SHUTDOWN_SEC = 5; - AwsIotConnectivityModule::AwsIotConnectivityModule( std::string rootCA, std::string clientId, - std::shared_ptr mqttClientBuilder ) + std::shared_ptr mqttClientBuilder, + AwsIotConnectivityConfig connectionConfig ) : mRootCA( std::move( rootCA ) ) , mClientId( std::move( clientId ) ) , mMqttClientBuilder( std::move( mqttClientBuilder ) ) - , mRetryThread( *this, RETRY_FIRST_CONNECTION_START_BACKOFF_MS, RETRY_FIRST_CONNECTION_MAX_BACKOFF_MS ) + , mConnectionConfig( std::move( connectionConfig ) ) + , mInitialConnectionThread( + [this]() -> RetryStatus { + return this->connectMqttClient(); + }, + RETRY_FIRST_CONNECTION_START_BACKOFF_MS, + RETRY_FIRST_CONNECTION_MAX_BACKOFF_MS ) + , mSubscriptionsThread( + [this]() -> RetryStatus { + return this->subscribeAllReceivers(); + }, + RETRY_FIRST_CONNECTION_START_BACKOFF_MS, + RETRY_FIRST_CONNECTION_MAX_BACKOFF_MS ) , mConnected( false ) , mConnectionEstablished( false ) { @@ -61,26 +56,49 @@ AwsIotConnectivityModule::connect() mConnected = false; FWE_LOG_INFO( "Establishing an MQTT Connection" ); - if ( !createMqttConnection() ) + if ( !createMqttClient() ) { return false; } - return mRetryThread.start(); + return mInitialConnectionThread.start(); } -std::shared_ptr -AwsIotConnectivityModule::createNewChannel( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription ) +std::shared_ptr +AwsIotConnectivityModule::createSender( const std::string &topicName, QoS publishQoS ) { - auto channel = std::make_shared( this, payloadManager, mMqttClient, topicName, subscription ); - mChannels.emplace_back( channel ); + auto sdkPublishQoS = Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE; + switch ( publishQoS ) { - std::lock_guard lock( mTopicToChannelMutex ); - mTopicToChannel[topicName] = channel; + case QoS::AT_MOST_ONCE: + sdkPublishQoS = Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE; + break; + case QoS::AT_LEAST_ONCE: + sdkPublishQoS = Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_LEAST_ONCE; + break; } - return channel; + + auto sender = std::make_shared( this, mMqttClient, topicName, sdkPublishQoS ); + mSenders.emplace_back( sender ); + return sender; +} + +std::shared_ptr +AwsIotConnectivityModule::createReceiver( const std::string &topicName ) +{ + auto receiver = std::make_shared( this, mMqttClient, topicName ); + mReceivers.emplace_back( receiver ); + + std::lock_guard lock( mTopicsMutex ); + mSubscribedTopicToReceiver[topicName] = receiver; + mSubscribedTopicsTree.insert( topicName, receiver ); + return receiver; +} + +void +AwsIotConnectivityModule::subscribeToConnectionEstablished( OnConnectionEstablishedCallback callback ) +{ + mConnectionEstablishedListeners.subscribe( callback ); } bool @@ -88,7 +106,7 @@ AwsIotConnectivityModule::resetConnection() { if ( !mConnectionEstablished ) { - return false; + return true; } mConnectionCompletedPromise = std::promise(); @@ -96,7 +114,15 @@ AwsIotConnectivityModule::resetConnection() // Get the future before calling the client code as get_future() is not guaranteed to be thread-safe auto closedResult = mConnectionClosedPromise.get_future(); FWE_LOG_INFO( "Closing the MQTT Connection" ); - if ( !mMqttClient->Stop() ) + // Before we close the connection, we need to create a Disconnect Packet + // So that we indicate to the broker that we intend to close the session, + // Otherwise, the socket will simply be closed and thus the broker thinks + // the connection is lost. + // Default constructor of DisconnectPacket sets the diconnectReason to + // CLIENT_INITIATED_DISCONNECT which is what we want here. + auto disconnectPacket = std::make_shared(); + + if ( !mMqttClient->Stop( disconnectPacket ) ) { FWE_LOG_ERROR( "Failed to close the MQTT Connection" ); return false; @@ -104,37 +130,50 @@ AwsIotConnectivityModule::resetConnection() closedResult.wait(); mConnectionEstablished = false; + return true; } bool AwsIotConnectivityModule::disconnect() { - // In case there is no connection or the connection is bad, we don't want to be waiting here for - // a long time. So we tell all channels to unsubscribe asynchronously, and then wait for them - // in a separate step. - std::vector>> unsubscribeResults; + mSubscriptionsThread.stop(); + + // Only unsubscribe from topics if not using persistent sessions. Otherwise when reconnecting + // the broker won't send messages sent while the client was disconnected. + if ( mConnectionConfig.sessionExpiryIntervalSeconds == 0 ) { - std::lock_guard lock( mTopicToChannelMutex ); - for ( auto &topicAndChannel : mTopicToChannel ) + // In case there is no connection or the connection is bad, we don't want to be waiting here for + // a long time. So we tell all receivers to unsubscribe asynchronously, and then wait for them + // in a separate step. + std::vector>> unsubscribeResults; { - unsubscribeResults.emplace_back( topicAndChannel.first, topicAndChannel.second->unsubscribeAsync() ); + std::lock_guard lock( mTopicsMutex ); + for ( auto &topicAndReceiver : mSubscribedTopicToReceiver ) + { + unsubscribeResults.emplace_back( topicAndReceiver.first, topicAndReceiver.second->unsubscribeAsync() ); + } } - } - auto timeout = std::chrono::steady_clock::now() + std::chrono::seconds( MQTT_UNSUBSCRIBE_TIMEOUT_ON_SHUTDOWN_SEC ); - for ( auto &topicAndResult : unsubscribeResults ) - { - if ( topicAndResult.second.wait_until( timeout ) == std::future_status::timeout ) + auto timeout = + std::chrono::steady_clock::now() + std::chrono::seconds( MQTT_UNSUBSCRIBE_TIMEOUT_ON_SHUTDOWN_SECONDS ); + for ( auto &topicAndResult : unsubscribeResults ) { - FWE_LOG_WARN( "Unsubscribe operation timed out for topic " + topicAndResult.first ); + if ( topicAndResult.second.wait_until( timeout ) == std::future_status::timeout ) + { + FWE_LOG_WARN( "Unsubscribe operation timed out for topic " + topicAndResult.first ); + } } } - mRetryThread.stop(); - for ( auto channel : mChannels ) + mInitialConnectionThread.stop(); + for ( auto sender : mSenders ) + { + sender->invalidateConnection(); + } + for ( auto receiver : mReceivers ) { - channel->invalidateConnection(); + receiver->invalidateConnection(); } return resetConnection(); } @@ -155,7 +194,7 @@ AwsIotConnectivityModule::renameEventLoopTask() } bool -AwsIotConnectivityModule::createMqttConnection() +AwsIotConnectivityModule::createMqttClient() { if ( mMqttClientBuilder == nullptr ) { @@ -173,15 +212,30 @@ AwsIotConnectivityModule::createMqttConnection() auto connectOptions = std::make_shared(); // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null connectOptions->WithClientId( mClientId.c_str() ) - .WithSessionExpiryIntervalSec( MQTT_SESSION_EXPIRY_INTERVAL_SEC ) - .WithKeepAliveIntervalSec( MQTT_CONNECT_KEEP_ALIVE_SECONDS ); + .WithSessionExpiryIntervalSec( mConnectionConfig.sessionExpiryIntervalSeconds ) + .WithKeepAliveIntervalSec( mConnectionConfig.keepAliveIntervalSeconds ); + + if ( mConnectionConfig.sessionExpiryIntervalSeconds > 0 ) + { + connectOptions->WithSessionExpiryIntervalSec( mConnectionConfig.sessionExpiryIntervalSeconds ); + } mMqttClientBuilder ->WithClientExtendedValidationAndFlowControl( - Aws::Crt::Mqtt5::ClientExtendedValidationAndFlowControl::AWS_MQTT5_EVAFCO_NONE ) + Aws::Crt::Mqtt5::ClientExtendedValidationAndFlowControl::AWS_MQTT5_EVAFCO_AWS_IOT_CORE_DEFAULTS ) + // Make queued packets fail on disconnection so we can better control how to handle those failures + // (e.g. drop, persist). Otherwise, using the default behavior, packets with QoS1 could stay in the + // queue for a long time and be transmitted even if they are no longer relevant. + .WithOfflineQueueBehavior( + Aws::Crt::Mqtt5::ClientOperationQueueBehaviorType::AWS_MQTT5_COQBT_FAIL_ALL_ON_DISCONNECT ) .WithConnectOptions( connectOptions ) - .WithSessionBehavior( Aws::Crt::Mqtt5::ClientSessionBehaviorType::AWS_MQTT5_CSBT_REJOIN_POST_SUCCESS ) - .WithPingTimeoutMs( MQTT_PING_TIMEOUT_MS ); + .WithPingTimeoutMs( mConnectionConfig.pingTimeoutMs ); + + if ( mConnectionConfig.sessionExpiryIntervalSeconds > 0 ) + { + mMqttClientBuilder->WithSessionBehavior( + Aws::Crt::Mqtt5::ClientSessionBehaviorType::AWS_MQTT5_CSBT_REJOIN_ALWAYS ); + } if ( !mRootCA.empty() ) { @@ -191,21 +245,50 @@ AwsIotConnectivityModule::createMqttConnection() mMqttClientBuilder->WithClientConnectionSuccessCallback( [&]( const Aws::Crt::Mqtt5::OnConnectionSuccessEventData &eventData ) { std::string logMessage = "Connection completed successfully"; + bool rejoinedSession = false; if ( eventData.negotiatedSettings != nullptr ) { + rejoinedSession = eventData.negotiatedSettings->getRejoinedSession(); // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null auto clientId = std::string( eventData.negotiatedSettings->getClientId().c_str() ); - logMessage += - ". ClientId: " + clientId + ", SessionExpiryIntervalSec: " + - std::to_string( eventData.negotiatedSettings->getSessionExpiryIntervalSec() ) + - ", ServerKeepAliveSec: " + std::to_string( eventData.negotiatedSettings->getServerKeepAlive() ) + - ", RejoinedSession: " + ( eventData.negotiatedSettings->getRejoinedSession() ? "true" : "false" ); + auto negotiatedKeepAlive = eventData.negotiatedSettings->getServerKeepAlive(); + logMessage += ". ClientId: " + clientId + ", SessionExpiryIntervalSec: " + + std::to_string( eventData.negotiatedSettings->getSessionExpiryIntervalSec() ) + + ", ServerKeepAliveSec: " + std::to_string( negotiatedKeepAlive ) + + ", RejoinedSession: " + ( rejoinedSession ? "true" : "false" ); + if ( negotiatedKeepAlive != mConnectionConfig.keepAliveIntervalSeconds ) + { + FWE_LOG_WARN( "Negotiated keep alive " + std::to_string( negotiatedKeepAlive ) + + " does not match the requested value " + + std::to_string( mConnectionConfig.keepAliveIntervalSeconds ) ); + } } FWE_LOG_INFO( logMessage ); mConnected = true; mConnectionCompletedPromise.set_value( true ); mConnectionCompletedPromise = std::promise(); renameEventLoopTask(); + + // If we didn't rejoin a session (which could happen because the previous session expired + // or persistent session is disabled), the client won't be subscribed to any topic even if + // this is a reconnection. So we need to ensure that all receivers subscribe again. + if ( !rejoinedSession ) + { + std::lock_guard lock( mTopicsMutex ); + for ( auto &topicAndReceiver : mSubscribedTopicToReceiver ) + { + topicAndReceiver.second->resetSubscription(); + } + } + + if ( mSubscriptionsThread.isAlive() ) + { + mSubscriptionsThread.restart(); + } + else + { + mSubscriptionsThread.start(); + } } ); mMqttClientBuilder->WithClientConnectionFailureCallback( @@ -231,7 +314,7 @@ AwsIotConnectivityModule::createMqttConnection() } ); mMqttClientBuilder->WithClientAttemptingConnectCallback( [&]( const OnAttemptingConnectEventData &eventData ) { - (void)eventData; + static_cast( eventData ); FWE_LOG_INFO( "Attempting MQTT connection" ); } ); @@ -261,7 +344,7 @@ AwsIotConnectivityModule::createMqttConnection() } ); mMqttClientBuilder->WithClientStoppedCallback( [&]( const Aws::Crt::Mqtt5::OnStoppedEventData &eventData ) { - (void)eventData; + static_cast( eventData ); FWE_LOG_INFO( "The MQTT connection is closed and client stopped" ); mConnectionClosedPromise.set_value(); mConnected = false; @@ -277,23 +360,19 @@ AwsIotConnectivityModule::createMqttConnection() // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null auto topic = std::string( eventData.publishPacket->getTopic().c_str() ); - std::shared_ptr channel; + std::shared_ptr receiver; { - std::lock_guard lock( mTopicToChannelMutex ); - auto it = mTopicToChannel.find( topic ); - if ( it != mTopicToChannel.end() ) - { - channel = it->second; - } + std::lock_guard lock( mTopicsMutex ); + receiver = mSubscribedTopicsTree.find( topic ); } - if ( channel == nullptr ) + if ( receiver == nullptr ) { - FWE_LOG_ERROR( "Channel not found for topic " + topic ); + FWE_LOG_ERROR( "Receiver not found for topic " + topic ); return; } - channel->onDataReceived( eventData ); + receiver->onDataReceived( eventData ); } ); TraceModule::get().sectionBegin( TraceSection::BUILD_MQTT ); @@ -320,7 +399,7 @@ AwsIotConnectivityModule::createMqttConnection() } RetryStatus -AwsIotConnectivityModule::attempt() +AwsIotConnectivityModule::connectMqttClient() { FWE_LOG_TRACE( "Starting MQTT client" ); @@ -330,41 +409,58 @@ AwsIotConnectivityModule::attempt() { int lastError = mMqttClient->LastError(); auto errorString = Aws::Crt::ErrorDebugString( lastError ); - FWE_LOG_WARN( "The MQTT Connection failed wit error code " + std::to_string( lastError ) + ": " + + FWE_LOG_WARN( "The MQTT Connection failed with error code " + std::to_string( lastError ) + ": " + std::string( errorString != nullptr ? errorString : "Unknown error" ) ); mConnectionCompletedPromise = std::promise(); return RetryStatus::RETRY; } - FWE_LOG_TRACE( "Waiting of connection completed callback" ); + FWE_LOG_TRACE( "Waiting for connection completed callback" ); mConnectionEstablished = true; // Block until the connection establishes or fails. // If the connection fails, the module will also fail. - if ( connectionResult.get() ) - { - return RetryStatus::SUCCESS; - } - else + if ( !connectionResult.get() ) { // Cleanup resources resetConnection(); return RetryStatus::RETRY; } + + return RetryStatus::SUCCESS; } -void -AwsIotConnectivityModule::onFinished( RetryStatus code ) +RetryStatus +AwsIotConnectivityModule::subscribeAllReceivers() { - if ( code == RetryStatus::SUCCESS ) + auto result = RetryStatus::SUCCESS; + + // We make a copy because the subscribe() call is blocking and can take very long. So we should + // not hold the lock the whole time, otherwise we could block the callbacks called by the MQTT + // client. + std::vector> receivers; { - for ( auto channel : mChannels ) + std::lock_guard lock( mTopicsMutex ); + for ( auto &topicAndReceiver : mSubscribedTopicToReceiver ) { - if ( channel->shouldSubscribeAsynchronously() ) - { - channel->subscribe(); - } + receivers.emplace_back( topicAndReceiver.second ); } } + + for ( auto receiver : receivers ) + { + if ( receiver->subscribe() != ConnectivityError::Success ) + { + result = RetryStatus::RETRY; + } + } + + if ( result == RetryStatus::SUCCESS ) + { + // subscribe to all topics first before notifying listeners for connection + mConnectionEstablishedListeners.notify(); + } + + return result; } AwsIotConnectivityModule::~AwsIotConnectivityModule() diff --git a/src/AwsIotConnectivityModule.h b/src/AwsIotConnectivityModule.h index 7315d4be..c2b97e83 100644 --- a/src/AwsIotConnectivityModule.h +++ b/src/AwsIotConnectivityModule.h @@ -3,19 +3,24 @@ #pragma once -#include "AwsIotChannel.h" -#include "IConnectivityChannel.h" +#include "AwsIotReceiver.h" +#include "AwsIotSender.h" #include "IConnectivityModule.h" +#include "IReceiver.h" +#include "ISender.h" +#include "Listener.h" +#include "LoggingModule.h" #include "MqttClientWrapper.h" -#include "PayloadManager.h" #include "RetryThread.h" #include +#include #include #include #include #include #include #include +#include #include namespace Aws @@ -23,13 +28,142 @@ namespace Aws namespace IoTFleetWise { +// Default MQTT Keep alive in seconds. +// Defines how often an MQTT PING message is sent to the MQTT broker to keep the connection alive +// Default set to 60 seconds. Every 60 seconds the stack will send an MQTT PING req. +// The longer this interval is, the more the stack takes to detect the state of the TCP connection +// at the lower network layers. +// This parameter is asserted in the C SDK. It shall be strictly bigger than the default +// connection ping timeout. +// Refer to https://github.com/awslabs/aws-c-mqtt/blob/a2ee9a321fcafa19b0473b88a54e0ae8dde5fddf/source/client.c#L1461 +// This default is the same as IoT Core's default. If you need to configure this, override it in the config file. +constexpr uint16_t MQTT_KEEP_ALIVE_INTERVAL_SECONDS = 1200; +// Default ping timeout value in milliseconds +// If a response is not received within this interval, the connection will be reestablished. +// If the PING request does not return within this interval, the stack will create a new one. +// This default is the same as aws-c-mqtt's default: +// https://github.com/awslabs/aws-c-mqtt/blob/a2ee9a321fcafa19b0473b88a54e0ae8dde5fddf/include/aws/mqtt/private/v5/mqtt5_utils.h#L82 +// If you need to configure this, override it in the config file. +constexpr uint32_t MQTT_PING_TIMEOUT_MS = 30000; +// Default expiry interval for persistent sessions. If 0, persistent sessions will be disabled. +constexpr uint32_t MQTT_SESSION_EXPIRY_INTERVAL_SECONDS = 0; + +// How much time to wait for a response to an unsubscribe operation when shutting the module down. +constexpr uint32_t MQTT_UNSUBSCRIBE_TIMEOUT_ON_SHUTDOWN_SECONDS = 5; + +struct AwsIotConnectivityConfig +{ + uint16_t keepAliveIntervalSeconds = MQTT_KEEP_ALIVE_INTERVAL_SECONDS; + uint32_t pingTimeoutMs = MQTT_PING_TIMEOUT_MS; + uint32_t sessionExpiryIntervalSeconds = MQTT_SESSION_EXPIRY_INTERVAL_SECONDS; +}; + +struct TopicNode +{ + std::shared_ptr mReceiver; + std::unordered_map> mChildren; +}; + +/** + * @brief Map topic filters, including wildcards, to receivers. + * + * This is necessary to provide a generic way to subscribe to topics containing wildcards. + * + * Differently from v3, the MQTT v5 client doesn't support setting callbacks per subscription. There + * is only one callback that receives all Publish packets and we need to do the routing ourselves. + * + * Since we need to use wildcards, we can't use a simple topic->receiver map. We need to split the + * topics and build a tree so that we can match level by level. + * + * For some related discussion, see https://github.com/aws/aws-iot-device-sdk-java-v2/issues/453 + */ +class TopicTree +{ +public: + void + insert( const std::string &topic, std::shared_ptr receiver ) + { + size_t start = 0; + size_t foundPos = topic.find( '/', start ); + TopicNode *currentNode = nullptr; + auto *currentChildren = &mChildren; + while ( true ) + { + size_t n = foundPos == std::string::npos ? foundPos : foundPos - start; + std::string topicLevel = topic.substr( start, n ); + currentNode = currentChildren->emplace( topicLevel, std::make_unique() ).first->second.get(); + currentChildren = ¤tNode->mChildren; + + if ( foundPos == std::string::npos ) + { + break; + } + + start = foundPos + 1; + foundPos = topic.find( '/', start ); + } + + if ( currentNode->mReceiver != nullptr ) + { + FWE_LOG_ERROR( "Topic already exists: " + topic ); + return; + } + + currentNode->mReceiver = receiver; + } + + std::shared_ptr + find( const std::string &topic ) + { + size_t start = 0; + size_t foundPos = topic.find( '/', start ); + TopicNode *currentNode = nullptr; + auto *currentChildren = &mChildren; + while ( true ) + { + size_t n = foundPos == std::string::npos ? foundPos : foundPos - start; + std::string topicLevel = topic.substr( start, n ); + auto it = currentChildren->find( topicLevel ); + // If we don't find an exact match, we need to check whether there is a wildcard + if ( it == currentChildren->end() ) + { + it = currentChildren->find( "+" ); + if ( it == currentChildren->end() ) + { + it = currentChildren->find( "#" ); + if ( it == currentChildren->end() ) + { + FWE_LOG_ERROR( "Topic level does not exist: " + topicLevel ); + return nullptr; + } + } + } + + currentNode = it->second.get(); + currentChildren = ¤tNode->mChildren; + + if ( ( foundPos == std::string::npos ) || ( it->first == "#" ) ) + { + break; + } + + start = foundPos + 1; + foundPos = topic.find( '/', start ); + } + return currentNode->mReceiver; + } + +private: + std::unordered_map> mChildren; +}; + /** * @brief bootstrap of the Aws Iot SDK. Only one object of this should normally exist * */ // coverity[cert_dcl60_cpp_violation] false positive - class only defined once // coverity[autosar_cpp14_m3_2_2_violation] false positive - class only defined once // coverity[misra_cpp_2008_rule_3_2_2_violation] false positive - class only defined once -class AwsIotConnectivityModule : public IRetryable, public IConnectivityModule +class AwsIotConnectivityModule : public IConnectivityModule { public: constexpr static uint32_t RETRY_FIRST_CONNECTION_START_BACKOFF_MS = 1000; // start retry after one second @@ -40,11 +174,13 @@ class AwsIotConnectivityModule : public IRetryable, public IConnectivityModule * * @param rootCA The Root CA for the certificate * @param clientId the id that is used to identify this connection instance - * @param mqttClientBuilder a buider that can create MQTT client instances + * @param mqttClientBuilder a builder that can create MQTT client instances + * @param connectionConfig allows some connection config to be overriden */ AwsIotConnectivityModule( std::string rootCA, std::string clientId, - std::shared_ptr mqttClientBuilder ); + std::shared_ptr mqttClientBuilder, + AwsIotConnectivityConfig connectionConfig = AwsIotConnectivityConfig() ); ~AwsIotConnectivityModule() override; AwsIotConnectivityModule( const AwsIotConnectivityModule & ) = delete; @@ -71,28 +207,16 @@ class AwsIotConnectivityModule : public IRetryable, public IConnectivityModule return mConnected; }; - RetryStatus attempt() override; + std::shared_ptr createSender( const std::string &topicName, QoS publishQoS = QoS::AT_MOST_ONCE ) override; - void onFinished( RetryStatus code ) override; + std::shared_ptr createReceiver( const std::string &topicName ) override; - /** - * @brief create a new channel sharing the connection of this module - * This call needs to be done before calling connect for all asynchronous subscribe channel - * @param payloadManager the payload manager used by the new channel, - * sending data - * @param topicName the topic which this channel should subscribe/publish to - * @param subscription whether the channel should subscribe to the topic. Otherwise it will - * just publish to it. - * - * @return a pointer to the newly created channel. A reference to the newly created channel is also hold inside this - * module. - */ - std::shared_ptr createNewChannel( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription = false ) override; + void subscribeToConnectionEstablished( OnConnectionEstablishedCallback callback ) override; private: - bool createMqttConnection(); + bool createMqttClient(); + RetryStatus connectMqttClient(); + RetryStatus subscribeAllReceivers(); static void renameEventLoopTask(); bool resetConnection(); @@ -100,16 +224,21 @@ class AwsIotConnectivityModule : public IRetryable, public IConnectivityModule std::string mClientId; std::shared_ptr mMqttClient; std::shared_ptr mMqttClientBuilder; - RetryThread mRetryThread; + AwsIotConnectivityConfig mConnectionConfig; + RetryThread mInitialConnectionThread; + RetryThread mSubscriptionsThread; std::promise mConnectionCompletedPromise; std::promise mConnectionClosedPromise; std::atomic mConnected; std::atomic mConnectionEstablished; + ThreadSafeListeners mConnectionEstablishedListeners; - std::vector> mChannels; - std::unordered_map> mTopicToChannel; - std::mutex mTopicToChannelMutex; + std::vector> mSenders; + std::vector> mReceivers; + std::unordered_map> mSubscribedTopicToReceiver; + TopicTree mSubscribedTopicsTree; + std::mutex mTopicsMutex; }; } // namespace IoTFleetWise diff --git a/src/AwsIotReceiver.cpp b/src/AwsIotReceiver.cpp new file mode 100644 index 00000000..9decc899 --- /dev/null +++ b/src/AwsIotReceiver.cpp @@ -0,0 +1,243 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "AwsIotReceiver.h" +#include "IConnectivityModule.h" +#include "LoggingModule.h" +#include "TimeTypes.h" +#include "TraceModule.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +AwsIotReceiver::AwsIotReceiver( IConnectivityModule *connectivityModule, + std::shared_ptr &mqttClient, + std::string topicName ) + : mConnectivityModule( connectivityModule ) + , mMqttClient( mqttClient ) + , mTopicName( std::move( topicName ) ) + , mSubscribed( false ) +{ +} + +AwsIotReceiver::~AwsIotReceiver() +{ + unsubscribe(); +} + +bool +AwsIotReceiver::isAlive() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + return isAliveNotThreadSafe(); +} + +bool +AwsIotReceiver::isAliveNotThreadSafe() +{ + if ( mConnectivityModule == nullptr ) + { + return false; + } + return mConnectivityModule->isAlive() && mSubscribed; +} + +ConnectivityError +AwsIotReceiver::subscribe() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + if ( mSubscribed ) + { + return ConnectivityError::Success; + } + + if ( mTopicName.empty() ) + { + FWE_LOG_ERROR( "Empty ingestion topic name provided" ); + return ConnectivityError::NotConfigured; + } + if ( !mConnectivityModule->isAlive() ) + { + FWE_LOG_ERROR( "MQTT Connection not established, failed to subscribe" ); + return ConnectivityError::NoConnection; + } + + /* + * Subscribe for incoming publish messages on topic. + */ + std::promise subscribeFinishedPromise; + auto onSubAck = [&]( int errorCode, std::shared_ptr subAckPacket ) { + mSubscribed = false; + if ( errorCode != 0 ) + { + TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); + auto errorString = Aws::Crt::ErrorDebugString( errorCode ); + FWE_LOG_ERROR( "Subscribe failed with error code " + std::to_string( errorCode ) + ": " + + std::string( errorString != nullptr ? errorString : "Unknown error" ) ); + subscribeFinishedPromise.set_value( false ); + return; + } + + std::string grantedQoS = "unknown"; + if ( subAckPacket != nullptr ) + { + for ( auto reasonCode : subAckPacket->getReasonCodes() ) + { + if ( reasonCode <= Aws::Crt::Mqtt5::SubAckReasonCode::AWS_MQTT5_SARC_GRANTED_QOS_2 ) + { + grantedQoS = std::to_string( reasonCode ); + } + else if ( reasonCode >= Aws::Crt::Mqtt5::SubAckReasonCode::AWS_MQTT5_SARC_UNSPECIFIED_ERROR ) + { + TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); + // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null + auto reasonString = std::string( subAckPacket->getReasonString().has_value() + ? subAckPacket->getReasonString()->c_str() + : "Unknown reason" ); + FWE_LOG_ERROR( "Server rejected subscription to topic " + mTopicName + ". Reason code " + + std::to_string( reasonCode ) + ": " + reasonString ); + subscribeFinishedPromise.set_value( false ); + // Just return on the first error found. There could be multiple reason codes if we request multiple + // subscriptions at once, but we always request a single one. + return; + } + } + } + + FWE_LOG_INFO( "Subscribe succeeded for topic " + mTopicName + " with QoS " + grantedQoS ); + mSubscribed = true; + subscribeFinishedPromise.set_value( true ); + }; + + FWE_LOG_TRACE( "Subscribing to topic " + mTopicName ); + // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null + Aws::Crt::Mqtt5::Subscription sub1( mTopicName.c_str(), Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_LEAST_ONCE ); + auto subPacket = std::make_shared(); + subPacket->WithSubscription( std::move( sub1 ) ); + + if ( !mMqttClient->Subscribe( subPacket, onSubAck ) ) + { + FWE_LOG_ERROR( "Subscribe failed for topic " + mTopicName ); + return ConnectivityError::NoConnection; + } + + // Blocked call until subscribe finished this call should quickly either fail or succeed but + // depends on the network quality the Bootstrap needs to retry subscribing if failed. + subscribeFinishedPromise.get_future().wait(); + + if ( !mSubscribed ) + { + return ConnectivityError::NoConnection; + } + + return ConnectivityError::Success; +} + +void +AwsIotReceiver::subscribeToDataReceived( OnDataReceivedCallback callback ) +{ + mListeners.subscribe( callback ); +} + +void +AwsIotReceiver::onDataReceived( const Aws::Crt::Mqtt5::PublishReceivedEventData &eventData ) +{ + Timestamp currentTime = mClock->monotonicTimeSinceEpochMs(); + auto mqttTopic = std::string( eventData.publishPacket->getTopic().c_str() ); + ReceivedConnectivityMessage receivedMessage{ + eventData.publishPacket->getPayload().ptr, eventData.publishPacket->getPayload().len, currentTime, mqttTopic }; + + mListeners.notify( receivedMessage ); +} + +bool +AwsIotReceiver::unsubscribe() +{ + auto result = unsubscribeAsync(); + result.wait(); + return result.get(); +} + +std::future +AwsIotReceiver::unsubscribeAsync() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + + // We can't move the promise into the lambda, because the lambda needs to be copyable. So we + // don't have much choice but use a shared pointer. + auto unsubscribeFinishedPromise = std::make_shared>(); + auto unsubscribeFuture = unsubscribeFinishedPromise->get_future(); + + if ( !isAliveNotThreadSafe() ) + { + unsubscribeFinishedPromise->set_value( false ); + return unsubscribeFuture; + } + + FWE_LOG_TRACE( "Unsubscribing..." ); + auto unsubPacket = std::make_shared(); + // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null + unsubPacket->WithTopicFilter( mTopicName.c_str() ); + mMqttClient->Unsubscribe( + unsubPacket, + [this, unsubscribeFinishedPromise]( int errorCode, std::shared_ptr unsubAckPacket ) { + if ( errorCode != 0 ) + { + auto errorString = Aws::Crt::ErrorDebugString( errorCode ); + std::string logMessage = "Unsubscribe failed with error code " + std::to_string( errorCode ) + ": " + + std::string( errorString != nullptr ? errorString : "Unknown error" ); + if ( errorCode == AWS_ERROR_MQTT5_USER_REQUESTED_STOP ) + { + FWE_LOG_TRACE( logMessage ); + } + else + { + TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); + FWE_LOG_ERROR( logMessage ); + } + unsubscribeFinishedPromise->set_value( false ); + return; + } + + if ( unsubAckPacket != nullptr ) + { + for ( Aws::Crt::Mqtt5::UnSubAckReasonCode reasonCode : unsubAckPacket->getReasonCodes() ) + { + if ( reasonCode >= Aws::Crt::Mqtt5::UnSubAckReasonCode::AWS_MQTT5_UARC_UNSPECIFIED_ERROR ) + { + // coverity[cert_str51_cpp_violation] - pointer comes from std::string, which can't be null + // Refer to the MQTT spec for details on the reason codes for errors. + // https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901179 + auto reasonString = std::string( unsubAckPacket->getReasonString().has_value() + ? unsubAckPacket->getReasonString()->c_str() + : "Refer to the MQTT Spec for details" ); + TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::SUBSCRIBE_ERROR ); + FWE_LOG_ERROR( "Server rejected unsubscribe from topic " + mTopicName + ". Reason code " + + std::to_string( reasonCode ) + ": " + reasonString ); + unsubscribeFinishedPromise->set_value( false ); + // Just return on the first error found. There could be multiple reason codes if we + // unsubscribe from multiple subscriptions at once, but we always request a single one. + return; + } + } + } + FWE_LOG_TRACE( "Unsubscribed from topic " + mTopicName ); + mSubscribed = false; + unsubscribeFinishedPromise->set_value( true ); + } ); + + return unsubscribeFuture; +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsIotReceiver.h b/src/AwsIotReceiver.h new file mode 100644 index 00000000..146eb07c --- /dev/null +++ b/src/AwsIotReceiver.h @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "Clock.h" +#include "ClockHandler.h" +#include "IConnectionTypes.h" +#include "IConnectivityModule.h" +#include "IReceiver.h" +#include "Listener.h" +#include "MqttClientWrapper.h" +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +/** + * @brief A receiver to receive messages using IoT Core MQTT connection + * + * There can be multiple AwsIotReceivers from one AwsIotConnectivityModule. The connection of the + * connectivityModule passed in the constructor must be established before anything meaningful + * can be done with this class. + * @see AwsIotConnectivityModule + */ +class AwsIotReceiver : public IReceiver +{ +public: + AwsIotReceiver( IConnectivityModule *connectivityModule, + std::shared_ptr &mqttClient, + std::string topicName ); + ~AwsIotReceiver() override; + + AwsIotReceiver( const AwsIotReceiver & ) = delete; + AwsIotReceiver &operator=( const AwsIotReceiver & ) = delete; + AwsIotReceiver( AwsIotReceiver && ) = delete; + AwsIotReceiver &operator=( AwsIotReceiver && ) = delete; + + /** + * @brief Subscribe to the MQTT topic from setTopic. Necessary if data is received on the topic + * + * This function blocks until subscribe succeeded or failed and should be done in the setup form + * the bootstrap thread. The connection of the connectivityModule passed in the constructor + * must be established otherwise subscribe will fail. No retries are done to try to subscribe + * this needs to be done in the bootstrap during the setup. + * @return Success if subscribe finished correctly + */ + ConnectivityError subscribe(); + + /** + * @brief After unsubscribe no data will be received over by the receiver + * @return True for success + */ + bool unsubscribe(); + + /** + * @brief Unsubscribe from the MQTT topic asynchronously + * @return A future that can be used to wait for the unsubscribe to finish. It will return True on success. + */ + std::future unsubscribeAsync(); + + bool isAlive() override; + void subscribeToDataReceived( OnDataReceivedCallback callback ) override; + + void onDataReceived( const Aws::Crt::Mqtt5::PublishReceivedEventData &eventData ); + + void + invalidateConnection() + { + std::lock_guard connectivityLock( mConnectivityMutex ); + mConnectivityModule = nullptr; + }; + + void + resetSubscription() + { + mSubscribed = false; + } + +private: + bool isAliveNotThreadSafe(); + + IConnectivityModule *mConnectivityModule; + ThreadSafeListeners mListeners; + std::shared_ptr &mMqttClient; + std::mutex mConnectivityMutex; + std::string mTopicName; + std::atomic mSubscribed; + + /** + * @brief Clock member variable used to generate the time an MQTT message was received + */ + std::shared_ptr mClock = ClockHandler::getClock(); +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsIotSender.cpp b/src/AwsIotSender.cpp new file mode 100644 index 00000000..349e3d10 --- /dev/null +++ b/src/AwsIotSender.cpp @@ -0,0 +1,144 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "AwsIotSender.h" +#include "AwsSDKMemoryManager.h" +#include "IConnectionTypes.h" +#include "IConnectivityModule.h" +#include "LoggingModule.h" +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +AwsIotSender::AwsIotSender( IConnectivityModule *connectivityModule, + std::shared_ptr &mqttClient, + std::string topicName, + Aws::Crt::Mqtt5::QOS publishQoS ) + : mConnectivityModule( connectivityModule ) + , mMqttClient( mqttClient ) + , mTopicName( std::move( topicName ) ) + , mPublishQoS( publishQoS ) +{ +} + +bool +AwsIotSender::isAlive() +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + return isAliveNotThreadSafe(); +} + +bool +AwsIotSender::isAliveNotThreadSafe() +{ + if ( mConnectivityModule == nullptr ) + { + return false; + } + return mConnectivityModule->isAlive(); +} + +size_t +AwsIotSender::getMaxSendSize() const +{ + return AWS_IOT_MAX_MESSAGE_SIZE; +} + +void +AwsIotSender::sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) +{ + sendBufferToTopic( mTopicName, buf, size, callback ); +} + +void +AwsIotSender::sendBufferToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ) +{ + std::lock_guard connectivityLock( mConnectivityMutex ); + if ( topic.empty() ) + { + FWE_LOG_WARN( "Invalid topic provided" ); + callback( ConnectivityError::NotConfigured ); + return; + } + + if ( ( buf == nullptr ) || ( size == 0 ) ) + { + FWE_LOG_WARN( "No valid data provided" ); + callback( ConnectivityError::WrongInputData ); + return; + } + + if ( size > getMaxSendSize() ) + { + FWE_LOG_WARN( "Payload provided is too long" ); + callback( ConnectivityError::WrongInputData ); + return; + } + + if ( !isAliveNotThreadSafe() ) + { + callback( ConnectivityError::NoConnection ); + return; + } + + if ( !AwsSDKMemoryManager::getInstance().reserveMemory( size ) ) + { + FWE_LOG_ERROR( "Not sending out the message with size " + std::to_string( size ) + + " because IoT device SDK allocated the maximum defined memory." ); + + callback( ConnectivityError::QuotaReached ); + return; + } + + publishMessageToTopic( topic, buf, size, callback ); +} + +void +AwsIotSender::publishMessageToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ) +{ + auto payload = Aws::Crt::ByteBufFromArray( buf, size ); + + auto onPublishComplete = [size, callback, this]( int errorCode, + std::shared_ptr result ) mutable { + { + AwsSDKMemoryManager::getInstance().releaseReservedMemory( size ); + } + + if ( result->wasSuccessful() ) + { + FWE_LOG_TRACE( "Publish succeeded" ); + mPayloadCountSent++; + callback( ConnectivityError::Success ); + } + else + { + auto errorString = Aws::Crt::ErrorDebugString( errorCode ); + FWE_LOG_ERROR( std::string( "Operation failed with error" ) + + ( errorString != nullptr ? std::string( errorString ) : std::string( "Unknown error" ) ) ); + callback( ConnectivityError::TransmissionError ); + } + }; + + std::shared_ptr publishPacket = std::make_shared( + topic.c_str(), Aws::Crt::ByteCursorFromByteBuf( payload ), mPublishQoS ); + if ( !mMqttClient->Publish( publishPacket, onPublishComplete ) ) + { + callback( ConnectivityError::TransmissionError ); + } +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsIotSender.h b/src/AwsIotSender.h new file mode 100644 index 00000000..c0423431 --- /dev/null +++ b/src/AwsIotSender.h @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "Clock.h" +#include "ClockHandler.h" +#include "IConnectivityModule.h" +#include "ISender.h" +#include "MqttClientWrapper.h" +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +/** + * @brief A sender to send messages using IoT Core MQTT connection + * + * There can be multiple AwsIotSenders from one AwsIotConnectivityModule. The connection of the + * connectivityModule passed in the constructor must be established before anything meaningful + * can be done with this class. + * @see AwsIotConnectivityModule + */ +class AwsIotSender : public ISender +{ +public: + AwsIotSender( IConnectivityModule *connectivityModule, + std::shared_ptr &mqttClient, + std::string topicName, + Aws::Crt::Mqtt5::QOS publishQoS ); + ~AwsIotSender() override = default; + + AwsIotSender( const AwsIotSender & ) = delete; + AwsIotSender &operator=( const AwsIotSender & ) = delete; + AwsIotSender( AwsIotSender && ) = delete; + AwsIotSender &operator=( AwsIotSender && ) = delete; + + bool isAlive() override; + + size_t getMaxSendSize() const override; + + void sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) override; + + void sendBufferToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ) override; + + void + invalidateConnection() + { + std::lock_guard connectivityLock( mConnectivityMutex ); + mConnectivityModule = nullptr; + }; + + /** + * @brief Returns the number of payloads successfully passed to the AWS IoT SDK + * @return Number of payloads + */ + unsigned + getPayloadCountSent() const override + { + return mPayloadCountSent; + } + +private: + bool isAliveNotThreadSafe(); + + // coverity[autosar_cpp14_a0_1_3_violation] false positive - function is used + bool + isTopicValid() + { + return !mTopicName.empty(); + }; + + void publishMessageToTopic( const std::string &topic, + const uint8_t *buf, + size_t size, + OnDataSentCallback callback ); + + /** See "Message size" : "The payload for every publish request can be no larger + * than 128 KB. AWS IoT Core rejects publish and connect requests larger than this size." + * https://docs.aws.amazon.com/general/latest/gr/iot-core.html#limits_iot + */ + static const size_t AWS_IOT_MAX_MESSAGE_SIZE = 131072; // = 128 KiB + IConnectivityModule *mConnectivityModule; + std::shared_ptr &mMqttClient; + std::mutex mConnectivityMutex; + std::string mTopicName; + std::atomic mPayloadCountSent{}; + + /** + * @brief Clock member variable used to generate the time an MQTT message was received + */ + std::shared_ptr mClock = ClockHandler::getClock(); + + Aws::Crt::Mqtt5::QOS mPublishQoS; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/AwsSDKMemoryManager.cpp b/src/AwsSDKMemoryManager.cpp index 1c6b8db9..7a578dac 100644 --- a/src/AwsSDKMemoryManager.cpp +++ b/src/AwsSDKMemoryManager.cpp @@ -4,23 +4,12 @@ #include "AwsSDKMemoryManager.h" #include "TraceModule.h" #include -#include namespace Aws { namespace IoTFleetWise { -using Byte = unsigned char; - -// offset to store value of memory size allocated -// Since different types have different alignment requirements, -// we use the largest available alignment offset. -// Note: This however does not work for over-aligned types -// https://en.cppreference.com/w/cpp/language/object#Alignment -// We are OK at the moment to not handled over-aligned types since we do not have any usage of "alignas" -constexpr auto ALIGN_OFFSET = alignof( std::max_align_t ); - AwsSDKMemoryManager & AwsSDKMemoryManager::getInstance() { @@ -28,73 +17,17 @@ AwsSDKMemoryManager::getInstance() return instance; } -void -AwsSDKMemoryManager::Begin() -{ -} - -void -AwsSDKMemoryManager::End() -{ -} - -void * -AwsSDKMemoryManager::AllocateMemory( std::size_t blockSize, std::size_t alignment, const char *allocationTag ) -{ - // suppress unused parameter errors - (void)alignment; - (void)allocationTag; - - // Verify that the object fits into the memory block - static_assert( ALIGN_OFFSET >= sizeof( std::size_t ), "too big memory size block" ); - - auto realSize = blockSize + ALIGN_OFFSET; - void *pMem = malloc( realSize ); // NOLINT(cppcoreguidelines-no-malloc) - - if ( pMem == nullptr ) - { - return nullptr; - } - - // store the allocated memory's size - *( static_cast( pMem ) ) = realSize; - mMemoryUsedAndReserved += realSize; - TraceModule::get().setVariable( TraceVariable::MQTT_HEAP_USAGE, mMemoryUsedAndReserved ); - - // return a pointer to the block offset from the size storage location - return static_cast( pMem ) + ALIGN_OFFSET; -} - -void -AwsSDKMemoryManager::FreeMemory( void *memoryPtr ) -{ - if ( memoryPtr == nullptr ) - { - return; - } - - // go back to the memory location where stored the size - auto pMem = static_cast( static_cast( memoryPtr ) - ALIGN_OFFSET ); - // read the size value - auto realSize = *( static_cast( pMem ) ); - - // free the memory - free( pMem ); // NOLINT(cppcoreguidelines-no-malloc) - - // update the stats - mMemoryUsedAndReserved -= realSize; - TraceModule::get().setVariable( TraceVariable::MQTT_HEAP_USAGE, mMemoryUsedAndReserved ); -} - std::size_t -AwsSDKMemoryManager::getLimit() const +AwsSDKMemoryManager::getLimit() { + std::lock_guard lock( mMutex ); return mMaximumAwsSDKMemorySize; } bool AwsSDKMemoryManager::setLimit( size_t size ) { + std::lock_guard lock( mMutex ); if ( size == 0U ) { return false; @@ -106,11 +39,12 @@ AwsSDKMemoryManager::setLimit( size_t size ) bool AwsSDKMemoryManager::reserveMemory( std::size_t bytes ) { - if ( ( mMemoryUsedAndReserved + bytes + ALIGN_OFFSET ) > mMaximumAwsSDKMemorySize ) + std::lock_guard lock( mMutex ); + if ( ( mMemoryUsedAndReserved + bytes ) > mMaximumAwsSDKMemorySize ) { return false; } - mMemoryUsedAndReserved += ( bytes + ALIGN_OFFSET ); + mMemoryUsedAndReserved += bytes; TraceModule::get().setVariable( TraceVariable::MQTT_HEAP_USAGE, mMemoryUsedAndReserved ); return true; } @@ -118,7 +52,8 @@ AwsSDKMemoryManager::reserveMemory( std::size_t bytes ) std::size_t AwsSDKMemoryManager::releaseReservedMemory( std::size_t bytes ) { - mMemoryUsedAndReserved -= ( bytes + ALIGN_OFFSET ); + std::lock_guard lock( mMutex ); + mMemoryUsedAndReserved -= bytes; TraceModule::get().setVariable( TraceVariable::MQTT_HEAP_USAGE, mMemoryUsedAndReserved ); return mMemoryUsedAndReserved; } diff --git a/src/AwsSDKMemoryManager.h b/src/AwsSDKMemoryManager.h index de4d1cf4..0f6eed5f 100644 --- a/src/AwsSDKMemoryManager.h +++ b/src/AwsSDKMemoryManager.h @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include #include +#include namespace Aws { @@ -11,26 +11,16 @@ namespace IoTFleetWise { /** - * @brief A minimal allocate implementation. - * In this memory manager, we store the size of the requested memory at the beginning of the allocated block - * and keep track of how much memory we are allocating and deallocating + * @brief Keep track of how much memory we are using for AWS SDK * - * NOTE: This allocator does not handle over-aligned types . See - * https://en.cppreference.com/w/cpp/language/object#Alignment. - * If you are using over-aligned types, do not use this allocator. + * Note that this is not an allocator. The methods of this class should be called before calling + * some SDK operation and after the operation succeeds or fails. */ -class AwsSDKMemoryManager : public Aws::Utils::Memory::MemorySystemInterface +class AwsSDKMemoryManager { public: static AwsSDKMemoryManager &getInstance(); - void Begin() override; - - void End() override; - - void *AllocateMemory( std::size_t blockSize, std::size_t alignment, const char *allocationTag = nullptr ) override; - - void FreeMemory( void *memoryPtr ) override; /** * @brief Set the Limit of maximal memory usage * @@ -45,7 +35,7 @@ class AwsSDKMemoryManager : public Aws::Utils::Memory::MemorySystemInterface * * @return std::size_t */ - std::size_t getLimit() const; + std::size_t getLimit(); /** * @brief Reserve a chunk of memory for usage later @@ -76,6 +66,8 @@ class AwsSDKMemoryManager : public Aws::Utils::Memory::MemorySystemInterface static constexpr std::size_t MAXIMUM_AWS_SDK_HEAP_MEMORY_BYTES = 10000000; size_t mMaximumAwsSDKMemorySize = MAXIMUM_AWS_SDK_HEAP_MEMORY_BYTES; + + std::mutex mMutex; }; } // namespace IoTFleetWise diff --git a/src/CANDataConsumer.cpp b/src/CANDataConsumer.cpp index 60ca6877..12fd090b 100644 --- a/src/CANDataConsumer.cpp +++ b/src/CANDataConsumer.cpp @@ -4,9 +4,11 @@ #include "CANDataConsumer.h" #include "CANDataTypes.h" #include "CANDecoder.h" +#include "CollectionInspectionAPITypes.h" #include "EnumUtility.h" #include "LoggingModule.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "TraceModule.h" #include #include @@ -21,8 +23,8 @@ namespace Aws namespace IoTFleetWise { -CANDataConsumer::CANDataConsumer( SignalBufferPtr signalBufferPtr ) - : mSignalBufferPtr{ std::move( signalBufferPtr ) } +CANDataConsumer::CANDataConsumer( SignalBufferDistributorPtr signalBufferDistributor ) + : mSignalBufferDistributor{ std::move( signalBufferDistributor ) } { } @@ -93,8 +95,9 @@ CANDataConsumer::processMessage( CANChannelNumericID channelId, // Create Collected Data Frame CollectedDataFrame collectedDataFrame; // Check if we want to collect RAW CAN Frame; If so we also need to ensure Buffer is valid - if ( ( mSignalBufferPtr.get() != nullptr ) && ( ( collectType == CANMessageCollectType::RAW ) || - ( collectType == CANMessageCollectType::RAW_AND_DECODE ) ) ) + if ( ( mSignalBufferDistributor.get() != nullptr ) && + ( ( collectType == CANMessageCollectType::RAW ) || + ( collectType == CANMessageCollectType::RAW_AND_DECODE ) ) ) { // prepare the raw CAN Frame struct CollectedCanRawFrame canRawFrame; @@ -108,8 +111,9 @@ CANDataConsumer::processMessage( CANChannelNumericID channelId, collectedDataFrame.mCollectedCanRawFrame = std::make_shared( canRawFrame ); } // check if we want to decode can frame into signals and collect signals - if ( ( mSignalBufferPtr.get() != nullptr ) && ( ( collectType == CANMessageCollectType::DECODE ) || - ( collectType == CANMessageCollectType::RAW_AND_DECODE ) ) ) + if ( ( mSignalBufferDistributor.get() != nullptr ) && + ( ( collectType == CANMessageCollectType::DECODE ) || + ( collectType == CANMessageCollectType::RAW_AND_DECODE ) ) ) { if ( format.isValid() ) { @@ -121,33 +125,11 @@ CANDataConsumer::processMessage( CANChannelNumericID channelId, for ( auto const &signal : decodedSignals ) { // Create Collected Signal Object - struct CollectedSignal collectedSignal; - const auto signalType = signal.mSignalType; - switch ( signalType ) - { - case SignalType::UINT64: - collectedSignal = CollectedSignal{ signal.mSignalID, - timestamp, - signal.mPhysicalValue.signalValue.uint64Val, - signal.mSignalType }; - break; - case SignalType::INT64: - collectedSignal = CollectedSignal{ signal.mSignalID, - timestamp, - signal.mPhysicalValue.signalValue.int64Val, - signal.mSignalType }; - break; - default: - collectedSignal = CollectedSignal{ signal.mSignalID, - timestamp, - signal.mPhysicalValue.signalValue.doubleVal, - signal.mSignalType }; - break; - } // Only add valid signals to the vector - if ( collectedSignal.signalID != INVALID_SIGNAL_ID ) + if ( signal.mSignalID != INVALID_SIGNAL_ID ) { - collectedSignalsGroup.push_back( collectedSignal ); + collectedSignalsGroup.push_back( CollectedSignal::fromDecodedSignal( + signal.mSignalID, timestamp, signal.mPhysicalValue, signal.mSignalType ) ); } } collectedDataFrame.mCollectedSignals = collectedSignalsGroup; @@ -169,33 +151,7 @@ CANDataConsumer::processMessage( CANChannelNumericID channelId, TraceModule::get().sectionEnd( traceSection ); - // Increase all queue metrics before pushing data to the buffer - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - - auto collectedSignals = collectedDataFrame.mCollectedSignals.size(); - TraceModule::get().addToAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignals ); - - bool canRawFrameCollected = collectedDataFrame.mCollectedCanRawFrame != nullptr; - if ( canRawFrameCollected ) - { - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_CAN ); - } - - if ( !mSignalBufferPtr->push( std::move( collectedDataFrame ) ) ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - - if ( canRawFrameCollected ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_CAN ); - } - - TraceModule::get().subtractFromAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignals ); - - FWE_LOG_WARN( "Signal buffer full" ); - } + mSignalBufferDistributor->push( std::move( collectedDataFrame ) ); } } diff --git a/src/CANDataConsumer.h b/src/CANDataConsumer.h index 66ab96c0..43cbd3b0 100644 --- a/src/CANDataConsumer.h +++ b/src/CANDataConsumer.h @@ -23,7 +23,7 @@ namespace IoTFleetWise class CANDataConsumer { public: - CANDataConsumer( SignalBufferPtr signalBufferPtr ); + CANDataConsumer( SignalBufferDistributorPtr signalBufferDistributor ); ~CANDataConsumer() = default; CANDataConsumer( const CANDataConsumer & ) = delete; @@ -50,7 +50,7 @@ class CANDataConsumer const CANDecoderDictionary::CANMsgDecoderMethodType &decoderMethod, CANMessageDecoderMethod ¤tMessageDecoderMethod ); - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; }; } // namespace IoTFleetWise diff --git a/src/CANDataSource.cpp b/src/CANDataSource.cpp index 79e9ffdf..ad037da4 100644 --- a/src/CANDataSource.cpp +++ b/src/CANDataSource.cpp @@ -194,7 +194,19 @@ CANDataSource::doWork( void *data ) int nmsgs = recvmmsg( dataSource->mSocket, &msg[0], PARALLEL_RECEIVED_FRAMES_FROM_KERNEL, 0, nullptr ); // coverity[autosar_cpp14_m19_3_1_violation] // coverity[misra_cpp_2008_rule_19_3_1_violation] errno needs to be used to recognize network down - FWE_GRACEFUL_FATAL_ASSERT( ( nmsgs != -1 ) || ( errno != ENETDOWN ), "Network interface went down", ); + FWE_GRACEFUL_FATAL_ASSERT( ( nmsgs != -1 ) || ( errno != ENODEV ), "Network interface was removed", ); + // coverity[autosar_cpp14_m19_3_1_violation] + // coverity[misra_cpp_2008_rule_19_3_1_violation] errno needs to be used to recognize network down + if ( ( nmsgs == -1 ) && ( ( errno == ENETDOWN ) || ( errno == ENETUNREACH ) ) ) + { + // coverity[autosar_cpp14_m19_3_1_violation] + // coverity[misra_cpp_2008_rule_19_3_1_violation] errno needs to be used to recognize network down + FWE_LOG_ERROR( "Network interface went down or unreachable with Syscall errno: " + + std::to_string( errno ) ); + // Not much to do here, Socket is still alive, when network is back, we continue to consume. + } + // Else, the socket is non blocking, so we might expect -1 as a nmsgs + for ( int i = 0; i < nmsgs; i++ ) { // After waking up the Socket Can old messages in the kernel queue need to be ignored diff --git a/src/CANDataTypes.h b/src/CANDataTypes.h index cae0ddd4..53c4c902 100644 --- a/src/CANDataTypes.h +++ b/src/CANDataTypes.h @@ -12,56 +12,19 @@ namespace Aws namespace IoTFleetWise { -union CANPhysicalValue { - double doubleVal; - uint64_t uint64Val; - int64_t int64Val; -}; - -struct CANPhysicalValueType -{ - CANPhysicalValue signalValue; - SignalType signalType; - - template - CANPhysicalValueType( T val, SignalType type ) - : signalType( type ) - { - switch ( signalType ) - { - case SignalType::UINT64: - signalValue.uint64Val = static_cast( val ); - break; - case SignalType::INT64: - signalValue.int64Val = static_cast( val ); - break; - default: - signalValue.doubleVal = static_cast( val ); - } - } - - SignalType - getType() const - { - return signalType; - } -}; - struct CANDecodedSignal { - CANDecodedSignal( uint32_t signalID, int64_t rawValue, CANPhysicalValueType physicalValue, SignalType signalTypeIn ) + CANDecodedSignal( uint32_t signalID, DecodedSignalValue physicalValue, SignalType signalTypeIn ) : mSignalID( signalID ) - , mRawValue( rawValue ) , mPhysicalValue( physicalValue ) , mSignalType( signalTypeIn ) { } uint32_t mSignalID; - int64_t mRawValue; - CANPhysicalValueType mPhysicalValue; - SignalType mSignalType{ SignalType::DOUBLE }; + DecodedSignalValue mPhysicalValue; + SignalType mSignalType{ SignalType::UNKNOWN }; }; /** diff --git a/src/CANDecoder.cpp b/src/CANDecoder.cpp index 39ac5133..3e93aff6 100644 --- a/src/CANDecoder.cpp +++ b/src/CANDecoder.cpp @@ -55,25 +55,22 @@ CANDecoder::decodeCANMessage( const uint8_t *frameData, " and type as uint64" ); } auto physicalRawValue = static_cast( multiplexorValue ); - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); - decodedSignals.emplace_back( - CANDecodedSignal( it->mSignalID, rawValue, physicalValue, CANsignalType ) ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); + decodedSignals.emplace_back( CANDecodedSignal( it->mSignalID, physicalValue, CANsignalType ) ); break; } case ( SignalType::INT64 ): { auto physicalRawValue = static_cast( multiplexorValue ); - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); - decodedSignals.emplace_back( - CANDecodedSignal( it->mSignalID, rawValue, physicalValue, CANsignalType ) ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); + decodedSignals.emplace_back( CANDecodedSignal( it->mSignalID, physicalValue, CANsignalType ) ); break; } default: { auto physicalRawValue = static_cast( multiplexorValue ); - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); - decodedSignals.emplace_back( - CANDecodedSignal( it->mSignalID, rawValue, physicalValue, CANsignalType ) ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); + decodedSignals.emplace_back( CANDecodedSignal( it->mSignalID, physicalValue, CANsignalType ) ); break; } } @@ -122,26 +119,26 @@ CANDecoder::decodeCANMessage( const uint8_t *frameData, uint64_t physicalRawValue = static_cast( rawValue ) * static_cast( format.mSignals[i].mFactor ) + static_cast( format.mSignals[i].mOffset ); - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); decodedSignals.emplace_back( - CANDecodedSignal( format.mSignals[i].mSignalID, rawValue, physicalValue, CANsignalType ) ); + CANDecodedSignal( format.mSignals[i].mSignalID, physicalValue, CANsignalType ) ); break; } case ( SignalType::INT64 ): { auto physicalRawValue = static_cast( rawValue ) * static_cast( format.mSignals[i].mFactor ) + static_cast( format.mSignals[i].mOffset ); - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); decodedSignals.emplace_back( - CANDecodedSignal( format.mSignals[i].mSignalID, rawValue, physicalValue, CANsignalType ) ); + CANDecodedSignal( format.mSignals[i].mSignalID, physicalValue, CANsignalType ) ); break; } default: { auto physicalRawValue = static_cast( rawValue ) * format.mSignals[i].mFactor + format.mSignals[i].mOffset; - auto physicalValue = CANPhysicalValueType( physicalRawValue, CANsignalType ); + auto physicalValue = DecodedSignalValue( physicalRawValue, CANsignalType ); decodedSignals.emplace_back( - CANDecodedSignal( format.mSignals[i].mSignalID, rawValue, physicalValue, CANsignalType ) ); + CANDecodedSignal( format.mSignals[i].mSignalID, physicalValue, CANsignalType ) ); } } } diff --git a/src/CANInterfaceIDTranslator.h b/src/CANInterfaceIDTranslator.h index 60608a8a..340d77fb 100644 --- a/src/CANInterfaceIDTranslator.h +++ b/src/CANInterfaceIDTranslator.h @@ -26,7 +26,7 @@ class CANInterfaceIDTranslator } CANChannelNumericID - getChannelNumericID( const InterfaceID &iid ) + getChannelNumericID( const InterfaceID &iid ) const { for ( auto l : mLookup ) { @@ -39,7 +39,7 @@ class CANInterfaceIDTranslator }; InterfaceID - getInterfaceID( CANChannelNumericID cid ) + getInterfaceID( CANChannelNumericID cid ) const { for ( auto l : mLookup ) { diff --git a/src/CPUUsageInfo.h b/src/CPUUsageInfo.h index d09d2787..1ce263be 100644 --- a/src/CPUUsageInfo.h +++ b/src/CPUUsageInfo.h @@ -3,7 +3,6 @@ #pragma once -#include #include #include #include diff --git a/src/CacheAndPersist.cpp b/src/CacheAndPersist.cpp index a51320aa..b0eed10a 100644 --- a/src/CacheAndPersist.cpp +++ b/src/CacheAndPersist.cpp @@ -22,11 +22,11 @@ CacheAndPersist::CacheAndPersist( const std::string &partitionPath, size_t maxPa , mPersistencyWorkspace{ partitionPath + ( ( ( partitionPath.empty() ) || ( partitionPath.back() == '/' ) ) ? "" : "/" ) + PERSISTENCY_WORKSPACE } - , mDecoderManifestFile{ mPersistencyWorkspace + DECODER_MANIFEST_FILE } - , mCollectionSchemeListFile{ mPersistencyWorkspace + COLLECTION_SCHEME_LIST_FILE } - , mPayloadMetadataFile{ mPersistencyWorkspace + PAYLOAD_METADATA_FILE } - , mCollectedDataPath{ mPersistencyWorkspace + COLLECTED_DATA_FOLDER } - , mMaxPersistencePartitionSize{ maxPartitionSize } + , mDecoderManifestFile( mPersistencyWorkspace + DECODER_MANIFEST_FILE ) + , mCollectionSchemeListFile( mPersistencyWorkspace + COLLECTION_SCHEME_LIST_FILE ) + , mPayloadMetadataFile( mPersistencyWorkspace + PAYLOAD_METADATA_FILE ) + , mCollectedDataPath( mPersistencyWorkspace + COLLECTED_DATA_FOLDER ) + , mMaxPersistencePartitionSize( maxPartitionSize ) { } @@ -92,6 +92,8 @@ CacheAndPersist::init() // Clean directory from files without metadata at startup cleanupPersistedData(); + // Write the metadata to ensure the file is created in case it doesn't exist yet + writeMetadata( mPersistedMetadata ); FWE_LOG_INFO( "Persistency library successfully initialised" ); return true; @@ -147,18 +149,9 @@ CacheAndPersist::write( const uint8_t *bufPtr, size_t size, std::string &path ) return ErrorCode::SUCCESS; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -/// @cond Ignore due to Doxygen bug. Even though ENABLE_PREPROCESSING is enabled, Doxygen warns -// about this overload not being declared when FWE_FEATURE_VISION_SYSTEM_DATA is disabled. ErrorCode -CacheAndPersist::write( std::unique_ptr streambuf, DataType dataType, const std::string &filename ) +CacheAndPersist::write( std::streambuf &streambuf, DataType dataType, const std::string &filename ) { - if ( streambuf == nullptr ) - { - FWE_LOG_ERROR( "Failed to persist data: stream is empty" ); - return ErrorCode::INVALID_DATA; - } - if ( filename.empty() ) { FWE_LOG_ERROR( "Failed to persist data: filename is empty" ); @@ -191,7 +184,7 @@ CacheAndPersist::write( std::unique_ptr streambuf, DataType data } std::ofstream file( mCollectedDataPath + filename, std::ios::binary ); - file << &( *streambuf ); + file << &streambuf; file.close(); if ( !file.good() ) @@ -202,11 +195,9 @@ CacheAndPersist::write( std::unique_ptr streambuf, DataType data return ErrorCode::SUCCESS; } -/// @endcond -#endif void -CacheAndPersist::addMetadata( Json::Value &metadata ) +CacheAndPersist::addMetadata( const Json::Value &metadata ) { mPersistedMetadata["files"].append( metadata ); } @@ -285,6 +276,33 @@ CacheAndPersist::read( uint8_t *const readBufPtr, size_t size, DataType dataType return read( readBufPtr, size, path ); } +ErrorCode +CacheAndPersist::read( std::ifstream &fileStream, DataType dataType, const std::string &filename ) +{ + std::string path = getFileName( dataType ); + if ( dataType == DataType::EDGE_TO_CLOUD_PAYLOAD ) + { + if ( filename.empty() ) + { + FWE_LOG_ERROR( "Failed to read persisted data: filename for the payload is empty " ); + return ErrorCode::INVALID_DATATYPE; + } + else + { + path += filename; + } + } + + fileStream.open( path, std::ios::in | std::ios::binary ); + if ( !fileStream.is_open() ) + { + FWE_LOG_ERROR( "Could not open file stream for file " + path ) + return ErrorCode::FILESYSTEM_ERROR; + } + + return ErrorCode::SUCCESS; +} + ErrorCode CacheAndPersist::read( uint8_t *const readBufPtr, size_t size, std::string &path ) const { @@ -487,7 +505,10 @@ CacheAndPersist::cleanupPersistedData() std::vector filenames; for ( const auto &file : mPersistedMetadata["files"] ) { - filenames.push_back( mCollectedDataPath + file["filename"].asString() ); + // Handle the legacy metadata. file["filename"] is the old one and file["payload"]["filename"] the new one + auto filename = + file["filename"].asString().empty() ? file["payload"]["filename"].asString() : file["filename"].asString(); + filenames.push_back( mCollectedDataPath + filename ); } // coverity[misra_cpp_2008_rule_14_8_2_violation] - boost filesystem path header defines both template and and // non-template function @@ -517,7 +538,7 @@ CacheAndPersist::cleanupPersistedData() // TODO: do not skip ion files but add the metadata for them so they don't get deleted // coverity[misra_cpp_2008_rule_14_8_2_violation] - boost filesystem path header defines both // template and and non-template function - if ( boost::filesystem::extension( filename ) != ".10n" ) // skip ion files + if ( it->path().extension() != ".10n" ) // skip ion files { filesToDelete.push_back( filename ); } @@ -533,6 +554,7 @@ CacheAndPersist::cleanupPersistedData() for ( auto &fileToDelete : filesToDelete ) { + FWE_LOG_TRACE( "Deleting file " + fileToDelete ); static_cast( erase( fileToDelete ) ); } diff --git a/src/CacheAndPersist.h b/src/CacheAndPersist.h index 89d8c373..5e6c2b79 100644 --- a/src/CacheAndPersist.h +++ b/src/CacheAndPersist.h @@ -6,12 +6,8 @@ #include #include #include -#include - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include #include -#endif +#include // max buffer to be allocated for a read buffer // this matches the Max Send Size on the AWS IoT channel @@ -41,7 +37,7 @@ enum class DataType PAYLOAD_METADATA, COLLECTION_SCHEME_LIST, DECODER_MANIFEST, - DEFAULT_DATA_TYPE + DEFAULT_DATA_TYPE, }; /** @@ -96,12 +92,11 @@ class CacheAndPersist DataType dataType, const std::string &filename = std::string() ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA /** - * @brief Writes to the non volatile memory(NVM) from stream to the Ion file. + * @brief Writes to the non volatile memory(NVM) from stream based on datatype and filename * - * @param streambuf data stream to write the data from - * @param dataType specifies if the data is an edge to cloud payload, collectionScheme list, etc. + * @param streambuf data stream to write the data from + * @param dataType specifies if the data is an edge to cloud payload, collectionScheme list, etc. * @param filename full name of the file to store * * @return ErrorCode SUCCESS if the write is successful, @@ -110,17 +105,14 @@ class CacheAndPersist * INVALID_DATATYPE if filename is empty, * FILESYSTEM_ERROR in case of any file I/O errors. */ - ErrorCode write( std::unique_ptr streambuf, - DataType dataType, - const std::string &filename = std::string() ); -#endif + ErrorCode write( std::streambuf &streambuf, DataType dataType, const std::string &filename = std::string() ); /** * @brief Adds new file metadata to existing JSON object. * * @param metadata JSON object of the file metadata to persist */ - void addMetadata( Json::Value &metadata ); + void addMetadata( const Json::Value &metadata ); /** * @brief Gets the size of data based on the datatype. @@ -158,6 +150,18 @@ class CacheAndPersist DataType dataType, const std::string &filename = std::string() ); + /** + * @brief Reads the persisted data of specified datatype in a file stream + * + * @param fileStream the file stream that will point to the file being requested + * @param dataType specifies if the data is an edge to cloud payload, collectionScheme list, etc. + * @param filename specifies file for the data to read, only valid for edge to cloud payload + * + * @return ErrorCode SUCCESS if the read is successful, + * INVALID_DATATYPE if provided datatype has no associated file, + */ + virtual ErrorCode read( std::ifstream &fileStream, DataType dataType, const std::string &filename = std::string() ); + /** * @brief Deletes persisted data for the specified data type and filename. * @param dataType specifies if the data is an edge to cloud payload, collectionScheme list, etc. diff --git a/src/CheckinSender.cpp b/src/CheckinSender.cpp new file mode 100644 index 00000000..727f7c6b --- /dev/null +++ b/src/CheckinSender.cpp @@ -0,0 +1,195 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "CheckinSender.h" +#include "LoggingModule.h" +#include "SignalTypes.h" +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +// Checkin retry interval. Used issue checkins to the cloud as soon as possible, set to 5 seconds +static constexpr uint32_t RETRY_CHECKIN_INTERVAL_IN_MILLISECOND = 5000; + +CheckinSender::CheckinSender( std::shared_ptr schemaListener, uint32_t checkinIntervalMs ) + : mSchemaListener( std::move( schemaListener ) ) +{ + if ( checkinIntervalMs > 0 ) + { + mCheckinIntervalMs = checkinIntervalMs; + } +} + +CheckinSender::~CheckinSender() +{ + if ( isAlive() ) + { + stop(); + } +} + +void +CheckinSender::onCheckinDocumentsChanged( const std::vector &documents ) +{ + std::lock_guard lock( mCheckinDocumentsMutex ); + mCheckinDocuments = documents; + mWait.notify(); +} + +bool +CheckinSender::start() +{ + // Prevent concurrent stop/init + std::lock_guard lock( mThreadMutex ); + // On multi core systems the shared variable mShouldStop must be updated for + // all cores before starting the thread otherwise thread will directly end + mShouldStop.store( false ); + if ( !mThread.create( + []( void *data ) { + CheckinSender *checkinSender = static_cast( data ); + checkinSender->doWork(); + }, + this ) ) + { + FWE_LOG_TRACE( "Checkin Thread failed to start" ); + } + else + { + FWE_LOG_TRACE( "Checkin Thread started" ); + mThread.setThreadName( "fwCheckin" ); + } + + return mThread.isActive() && mThread.isValid(); +} + +void +CheckinSender::doWork() +{ + setTimeToSendNextCheckin( mClock->monotonicTimeSinceEpochMs() ); + + while ( true ) + { + if ( shouldStop() ) + { + break; + } + + auto timeToSendNextCheckinMs = getTimeToSendNextCheckin(); + + if ( !timeToSendNextCheckinMs.has_value() ) + { + auto timeToWaitMs = mCheckinIntervalMs; + FWE_LOG_TRACE( "Spurious wake up. Still waiting for the previous checkin response.Sleeping again for: " + + std::to_string( timeToWaitMs ) + " ms" ); + mWait.wait( timeToWaitMs ); + continue; + } + + auto currentTimeMs = mClock->monotonicTimeSinceEpochMs(); + + if ( timeToSendNextCheckinMs.get() > currentTimeMs ) + { + auto timeToWaitMs = static_cast( timeToSendNextCheckinMs.get() - currentTimeMs ); + FWE_LOG_TRACE( "Spurious wake up. Time for next checkin not reached yet. Sleeping again for: " + + std::to_string( timeToWaitMs ) + " ms" ); + mWait.wait( timeToWaitMs ); + continue; + } + + { + std::unique_lock lock( mCheckinDocumentsMutex ); + + if ( !mCheckinDocuments.has_value() ) + { + lock.unlock(); + FWE_LOG_TRACE( "List of checkin documents not available yet. Sleeping until it is available" ); + mWait.wait( Signal::WaitWithPredicate ); + continue; + } + + auto &checkinDocuments = mCheckinDocuments.get(); + + std::string checkinLogStr; + for ( size_t i = 0; i < checkinDocuments.size(); i++ ) + { + if ( i > 0 ) + { + checkinLogStr += ", "; + } + checkinLogStr += checkinDocuments[i]; + } + FWE_LOG_TRACE( "CHECKIN: " + checkinLogStr ); + + setTimeToSendNextCheckin( boost::none ); + mSchemaListener->sendCheckin( checkinDocuments, [this, currentTimeMs]( bool success ) { + if ( success ) + { + setTimeToSendNextCheckin( currentTimeMs + mCheckinIntervalMs ); + } + else + { + uint32_t minimumCheckinInterval = + std::min( RETRY_CHECKIN_INTERVAL_IN_MILLISECOND, mCheckinIntervalMs ); + setTimeToSendNextCheckin( mClock->monotonicTimeSinceEpochMs() + minimumCheckinInterval ); + mWait.notify(); + } + } ); + } + + mWait.wait( mCheckinIntervalMs ); + } +} + +boost::optional +CheckinSender::getTimeToSendNextCheckin() +{ + std::lock_guard lock( mTimeToSendNextCheckinMutex ); + return mTimeToSendNextCheckin; +} + +void +CheckinSender::setTimeToSendNextCheckin( boost::optional timeToSendNextCheckin ) +{ + std::lock_guard lock( mTimeToSendNextCheckinMutex ); + mTimeToSendNextCheckin = timeToSendNextCheckin; +} + +bool +CheckinSender::stop() +{ + if ( ( !mThread.isValid() ) || ( !mThread.isActive() ) ) + { + return true; + } + std::lock_guard lock( mThreadMutex ); + mShouldStop.store( true, std::memory_order_relaxed ); + FWE_LOG_TRACE( "Request stop" ); + mWait.notify(); + mThread.release(); + FWE_LOG_TRACE( "Stop finished" ); + mShouldStop.store( false, std::memory_order_relaxed ); + return !mThread.isActive(); +} + +bool +CheckinSender::shouldStop() const +{ + return mShouldStop.load( std::memory_order_relaxed ); +} + +bool +CheckinSender::isAlive() +{ + return mThread.isValid() && mThread.isActive(); +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/CheckinSender.h b/src/CheckinSender.h new file mode 100644 index 00000000..f5650dc9 --- /dev/null +++ b/src/CheckinSender.h @@ -0,0 +1,101 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "Clock.h" +#include "ClockHandler.h" +#include "SchemaListener.h" +#include "Signal.h" +#include "SignalTypes.h" +#include "Thread.h" +#include "TimeTypes.h" +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +/** + * @brief This thread sends the check-in messages to the Cloud + * + * For that to happen it needs to be notified whenever any of the documents relevant to check-in + * changes. + */ +class CheckinSender +{ +public: + CheckinSender( std::shared_ptr schemaListener, uint32_t checkinIntervalMs = 0 ); + ~CheckinSender(); + + CheckinSender( const CheckinSender & ) = delete; + CheckinSender &operator=( const CheckinSender & ) = delete; + CheckinSender( CheckinSender && ) = delete; + CheckinSender &operator=( CheckinSender && ) = delete; + + /** + * @brief Callback from CollectionSchemeManager to notify that the new scheme is available + * + * This needs to be called at least once, otherwise no checkin message will be sent. + * + * @param documents the list of documents that will be sent in the next checkin message + * */ + void onCheckinDocumentsChanged( const std::vector &documents ); + + /** + * @brief stops the internal thread if started and wait until it finishes + * + * @return true if the stop was successful + */ + bool stop(); + + /** + * @brief starts the internal thread + * + * @return true if the start was successful + */ + bool start(); + + /** + * @brief Checks that the worker thread is healthy. + */ + bool isAlive(); + +private: + // default checkin interval set to 5 mins + static constexpr uint32_t DEFAULT_CHECKIN_INTERVAL_IN_MILLISECOND = 300000; + + bool shouldStop() const; + + void doWork(); + + boost::optional getTimeToSendNextCheckin(); + void setTimeToSendNextCheckin( boost::optional timeToSendNextCheckin ); + + // Time interval in ms to send checkin message + uint32_t mCheckinIntervalMs{ DEFAULT_CHECKIN_INTERVAL_IN_MILLISECOND }; + boost::optional mTimeToSendNextCheckin; + std::mutex mTimeToSendNextCheckinMutex; + + Thread mThread; + std::atomic mShouldStop{ false }; + std::mutex mThreadMutex; + Signal mWait; + std::shared_ptr mClock = ClockHandler::getClock(); + + // The list of checkin documents that will be sent in the next checkin message. If this optional + // doesn't contain a value, then no checkin message will be sent. + boost::optional> mCheckinDocuments; + std::mutex mCheckinDocumentsMutex; + + std::shared_ptr mSchemaListener; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/CollectionInspectionAPITypes.h b/src/CollectionInspectionAPITypes.h index bf0dae99..314f529e 100644 --- a/src/CollectionInspectionAPITypes.h +++ b/src/CollectionInspectionAPITypes.h @@ -4,14 +4,18 @@ #pragma once #include "CANDataTypes.h" +#include "DataSenderTypes.h" #include "EventTypes.h" +#include "Listener.h" +#include "LoggingModule.h" #include "MessageTypes.h" #include "OBDDataTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" -#include -#include +#include "TraceModule.h" +#include +#include #include - namespace Aws { namespace IoTFleetWise @@ -37,43 +41,46 @@ struct PassThroughMetadata bool compress{ false }; bool persist{ false }; uint32_t priority{ 0 }; - std::string decoderID; - std::string collectionSchemeID; + SyncID decoderID; + SyncID collectionSchemeID; }; // As a start these structs are mainly a copy of the data defined in ICollectionScheme but as plain old data structures struct InspectionMatrixSignalCollectionInfo { SignalID signalID; - uint32_t sampleBufferSize; /**< at least this amount of last x samples will be kept in buffer*/ + size_t sampleBufferSize; /**< at least this amount of last x samples will be kept in buffer*/ uint32_t minimumSampleIntervalMs; /**< zero means all signals are recorded as seen on the bus */ uint32_t fixedWindowPeriod; /**< zero means no fixed window sampling would happen */ bool isConditionOnlySignal; /**< Should the collected signals be sent to cloud or are the number * of samples in the buffer only necessary for condition evaluation */ - SignalType signalType{ SignalType::DOUBLE }; + SignalType signalType{ SignalType::UNKNOWN }; }; struct InspectionMatrixCanFrameCollectionInfo { CANRawFrameID frameID; CANChannelNumericID channelID; - uint32_t sampleBufferSize; /**< at least this amount of last x raw can frames will be kept in buffer */ + size_t sampleBufferSize; /**< at least this amount of last x raw can frames will be kept in buffer */ uint32_t minimumSampleIntervalMs; /**< 0 which all frames are recording as seen on the bus */ }; struct ConditionWithCollectedData { - const ExpressionNode *condition; /**< points into InspectionMatrix.expressionNodes; - * Raw pointer is used as needed for efficient AST and ConditionWithCollectedData - * never exists without the relevant InspectionMatrix */ - uint32_t minimumPublishIntervalMs; - uint32_t afterDuration; + const ExpressionNode *condition = + nullptr; /**< points into InspectionMatrix.expressionNodes; + * Raw pointer is used as needed for efficient AST and ConditionWithCollectedData + * never exists without the relevant InspectionMatrix */ + uint32_t minimumPublishIntervalMs{}; + uint32_t afterDuration{}; std::vector signals; std::vector canFrames; - bool includeActiveDtcs; - bool triggerOnlyOnRisingEdge; + bool includeActiveDtcs{}; + bool triggerOnlyOnRisingEdge{}; PassThroughMetadata metadata; + bool isStaticCondition{ true }; + bool alwaysEvaluateCondition{ false }; }; struct InspectionMatrix @@ -85,6 +92,97 @@ struct InspectionMatrix * locality). The traversal is depth first preorder */ }; +struct InspectionValue +{ + InspectionValue() = default; + InspectionValue( const InspectionValue & ) = delete; + InspectionValue &operator=( const InspectionValue & ) = delete; + InspectionValue( InspectionValue && ) = default; + InspectionValue &operator=( InspectionValue && ) = delete; + ~InspectionValue() = default; + enum class DataType + { + UNDEFINED, + BOOL, + DOUBLE, + }; + DataType type = DataType::UNDEFINED; + bool boolVal{}; + double doubleVal{}; + InspectionValue & + operator=( bool val ) + { + boolVal = val; + type = DataType::BOOL; + return *this; + } + InspectionValue & + operator=( double val ) + { + doubleVal = val; + type = DataType::DOUBLE; + return *this; + } + InspectionValue & + operator=( int val ) + { + *this = static_cast( val ); + return *this; + } + bool + isBoolOrDouble() const + { + return ( type == DataType::BOOL ) || ( type == DataType::DOUBLE ); + } + double + asDouble() const + { + return ( type == DataType::BOOL ) ? ( boolVal ? 1.0 : 0.0 ) : doubleVal; + } + bool + asBool() const + { + return ( type == DataType::DOUBLE ) ? ( doubleVal != 0.0 ) : boolVal; + } +}; + +struct SampleConsumed +{ + bool + isAlreadyConsumed( uint32_t conditionId ) + { + return mAlreadyConsumed.test( conditionId ); + } + void + setAlreadyConsumed( uint32_t conditionId, bool value ) + { + if ( conditionId == ALL_CONDITIONS ) + { + if ( value ) + { + mAlreadyConsumed.set(); + } + else + { + mAlreadyConsumed.reset(); + } + } + else + { + mAlreadyConsumed[conditionId] = value; + } + } + +private: + std::bitset mAlreadyConsumed{ 0 }; +}; +template +struct SignalSample : SampleConsumed +{ + T mValue; + Timestamp mTimestamp{ 0 }; +}; + // These values are provided by the CANDataConsumers struct CollectedCanRawFrame { @@ -120,6 +218,13 @@ union SignalValue { uint32_t uint32Val; int32_t int32Val; uint64_t uint64Val; + struct RawDataVal + { + uint32_t signalId; + uint32_t handle; + }; + RawDataVal rawDataVal; + SignalValue & operator=( const uint8_t value ) { @@ -193,12 +298,18 @@ union SignalValue { boolVal = value; return *this; } + SignalValue & + operator=( const RawDataVal value ) + { + rawDataVal = value; + return *this; + } }; struct SignalValueWrapper { SignalValue value{ 0 }; - SignalType type{ SignalType::DOUBLE }; + SignalType type{ SignalType::UNKNOWN }; SignalValueWrapper() = default; ~SignalValueWrapper() = default; @@ -239,15 +350,6 @@ struct CollectedSignal CollectedSignal() = default; - // Backward Compatibility - template - CollectedSignal( SignalID signalIDIn, Timestamp receiveTimeIn, T sigValue ) - : signalID( signalIDIn ) - , receiveTime( receiveTimeIn ) - { - value.setVal( static_cast( sigValue ), SignalType::DOUBLE ); - } - template CollectedSignal( SignalID signalIDIn, Timestamp receiveTimeIn, T sigValue, SignalType sigType ) : signalID( signalIDIn ) @@ -288,14 +390,34 @@ struct CollectedSignal case SignalType::BOOLEAN: value.setVal( static_cast( sigValue ), sigType ); break; + case SignalType::UNKNOWN: + // Signal of type UNKNOWN will not be collected + break; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: + case SignalType::COMPLEX_SIGNAL: value.setVal( static_cast( sigValue ), sigType ); break; #endif } } + static CollectedSignal + fromDecodedSignal( SignalID signalIDIn, + Timestamp receiveTimeIn, + const DecodedSignalValue &decodedSignalValue, + SignalType signalType ) + { + switch ( decodedSignalValue.signalType ) + { + case SignalType::UINT64: + return CollectedSignal{ signalIDIn, receiveTimeIn, decodedSignalValue.signalValue.uint64Val, signalType }; + case SignalType::INT64: + return CollectedSignal{ signalIDIn, receiveTimeIn, decodedSignalValue.signalValue.int64Val, signalType }; + default: + return CollectedSignal{ signalIDIn, receiveTimeIn, decodedSignalValue.signalValue.doubleVal, signalType }; + } + } + SignalType getType() const { @@ -350,72 +472,17 @@ struct CollectedDataFrame DTCInfoPtr mActiveDTCs; }; -// Thread-safe queue with mutex -template -struct LockedQueue -{ -public: - LockedQueue( size_t maxSize ) - : mMaxSize( maxSize ) - { - } - bool - push( const T &&element ) - { - std::lock_guard lock( mMutex ); - if ( ( mQueue.size() + 1 ) > mMaxSize ) - { - return false; - } - mQueue.push( element ); - return true; - } - bool - pop( T &element ) - { - std::lock_guard lock( mMutex ); - if ( mQueue.empty() ) - { - return false; - } - element = mQueue.front(); - mQueue.pop(); - return true; - } - template - size_t - consumeAll( const Functor &functor ) - { - size_t consumed = 0; - T element; - while ( pop( element ) ) - { - functor( element ); - consumed++; - } - return consumed; - } - // coverity[misra_cpp_2008_rule_14_7_1_violation] Required in unit tests - bool - isEmpty() - { - std::lock_guard lock( mMutex ); - return mQueue.empty(); - } - -private: - std::mutex mMutex; - size_t mMaxSize; - std::queue mQueue; -}; - // Buffer that sends data to Collection Engine using SignalBuffer = LockedQueue; // Shared Pointer type to the buffer that sends data to Collection Engine +// coverity[misra_cpp_2008_rule_0_1_5_violation] definition needed for tests +// coverity[autosar_cpp14_a0_1_6_violation] same using SignalBufferPtr = std::shared_ptr; +using SignalBufferDistributor = LockedQueueDistributor; +using SignalBufferDistributorPtr = std::shared_ptr; // Output of collection Inspection Engine -struct TriggeredCollectionSchemeData +struct TriggeredCollectionSchemeData : DataToSend { PassThroughMetadata metadata; Timestamp triggerTime; @@ -426,10 +493,52 @@ struct TriggeredCollectionSchemeData #endif DTCInfo mDTCInfo; EventID eventID; + + ~TriggeredCollectionSchemeData() override = default; + + SenderDataType + getDataType() const override + { + return SenderDataType::TELEMETRY; + } }; -using TriggeredCollectionSchemeDataPtr = std::shared_ptr; -using CollectedDataReadyToPublish = LockedQueue; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +struct TriggeredVisionSystemData : DataToSend +{ + PassThroughMetadata metadata; + Timestamp triggerTime; + std::vector signals; + EventID eventID; + + ~TriggeredVisionSystemData() override = default; + + SenderDataType + getDataType() const override + { + return SenderDataType::VISION_SYSTEM; + } +}; +#endif + +struct CollectionInspectionEngineOutput +{ + std::shared_ptr triggeredCollectionSchemeData; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::shared_ptr triggeredVisionSystemData; +#endif +}; + +enum class ExpressionErrorCode +{ + SUCCESSFUL, + SIGNAL_NOT_FOUND, + FUNCTION_DATA_NOT_AVAILABLE, + STACK_DEPTH_REACHED, + NOT_IMPLEMENTED_TYPE, + NOT_IMPLEMENTED_FUNCTION, + TYPE_MISMATCH +}; } // namespace IoTFleetWise } // namespace Aws diff --git a/src/CollectionInspectionEngine.cpp b/src/CollectionInspectionEngine.cpp index 0986fe39..16c6ce66 100644 --- a/src/CollectionInspectionEngine.cpp +++ b/src/CollectionInspectionEngine.cpp @@ -17,57 +17,35 @@ CollectionInspectionEngine::CollectionInspectionEngine( bool sendDataOnlyOncePer setActiveDTCsConsumed( ALL_CONDITIONS, false ); } -bool -CollectionInspectionEngine::isSignalPartOfEval( const ExpressionNode *expression, - InspectionSignalID signalID, - int remainingStackDepth ) -{ - if ( ( remainingStackDepth <= 0 ) || ( expression == nullptr ) ) - { - return false; - } - if ( ( expression->nodeType == ExpressionNodeType::SIGNAL ) || - ( expression->nodeType == ExpressionNodeType::WINDOWFUNCTION ) ) - { - return expression->signalID == signalID; - } - // Recursion limited depth through last parameter - bool leftRet = isSignalPartOfEval( expression->left, signalID, remainingStackDepth - 1 ); - bool rightRet = isSignalPartOfEval( expression->right, signalID, remainingStackDepth - 1 ); - return leftRet || rightRet; -} - template void -CollectionInspectionEngine::addSignalToBuffer( const InspectionMatrixSignalCollectionInfo &signalIn ) +CollectionInspectionEngine::addSignalBuffer( const InspectionMatrixSignalCollectionInfo &signal ) { - std::vector> *signalHistoryBufferPtr = nullptr; - const auto signalIDIn = signalIn.signalID; - - signalHistoryBufferPtr = getSignalHistoryBufferPtr( signalIDIn ); + auto signalHistoryBufferVectorPtr = getSignalHistoryBuffersPtr( signal.signalID ); - if ( signalHistoryBufferPtr == nullptr ) + if ( signalHistoryBufferVectorPtr == nullptr ) { return; } - auto &bufferVec = *signalHistoryBufferPtr; - for ( auto &buffer : bufferVec ) + auto &signalHistoryBufferVector = *signalHistoryBufferVectorPtr; + for ( auto &buffer : signalHistoryBufferVector ) { - if ( buffer.mMinimumSampleIntervalMs == signalIn.minimumSampleIntervalMs ) + // There is one buffer per sample interval ms for each signal + if ( buffer.mMinimumSampleIntervalMs == signal.minimumSampleIntervalMs ) { - buffer.mSize = std::max( buffer.mSize, signalIn.sampleBufferSize ); - buffer.addFixedWindow( signalIn.fixedWindowPeriod ); + buffer.mSize = std::max( buffer.mSize, signal.sampleBufferSize ); + buffer.addFixedWindow( signal.fixedWindowPeriod ); return; } } - bufferVec.emplace_back( signalIn.sampleBufferSize, - signalIn.minimumSampleIntervalMs + signalHistoryBufferVector.emplace_back( signal.sampleBufferSize, + signal.minimumSampleIntervalMs #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - signalIn.signalType == SignalType::RAW_DATA_BUFFER_HANDLE + , + signal.signalType == SignalType::COMPLEX_SIGNAL #endif ); - bufferVec.back().addFixedWindow( signalIn.fixedWindowPeriod ); + signalHistoryBufferVector.back().addFixedWindow( signal.fixedWindowPeriod ); } void @@ -76,10 +54,13 @@ CollectionInspectionEngine::onChangeInspectionMatrix( const std::shared_ptrconditions ) + for ( auto &condition : mActiveInspectionMatrix->conditions ) { // Check if we can add an additional condition to mConditions if ( mConditions.size() >= MAX_NUMBER_OF_ACTIVE_CONDITION ) @@ -91,8 +72,7 @@ CollectionInspectionEngine::onChangeInspectionMatrix( const std::shared_ptr MAX_DIFFERENT_SIGNAL_IDS ) + if ( condition.signals.size() > MAX_DIFFERENT_SIGNAL_IDS ) { TraceModule::get().incrementVariable( TraceVariable::CE_SIGNAL_ID_OUTBOUND ); FWE_LOG_ERROR( "There can be only " + std::to_string( MAX_DIFFERENT_SIGNAL_IDS ) + @@ -100,207 +80,217 @@ CollectionInspectionEngine::onChangeInspectionMatrix( const std::shared_ptr( s ); + addSignalBuffer( signal ); break; case SignalType::INT8: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::UINT16: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::INT16: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::UINT32: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::INT32: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::UINT64: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::INT64: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::FLOAT: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::DOUBLE: - addSignalToBuffer( s ); + addSignalBuffer( signal ); break; case SignalType::BOOLEAN: - addSignalToBuffer( s ); + addSignalBuffer( signal ); + break; + case SignalType::UNKNOWN: + // Signal of UNKNOWN type should not be processed break; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: - addSignalToBuffer( s ); + case SignalType::COMPLEX_SIGNAL: + addSignalBuffer( signal ); break; #endif } } - for ( auto &c : p.canFrames ) + for ( auto &canFrame : condition.canFrames ) { bool found = false; for ( auto &buf : mCanFrameBuffers ) { - if ( ( buf.mFrameID == c.frameID ) && ( buf.mChannelID == c.channelID ) && - ( buf.mMinimumSampleIntervalMs == c.minimumSampleIntervalMs ) ) + if ( ( buf.mFrameID == canFrame.frameID ) && ( buf.mChannelID == canFrame.channelID ) && + ( buf.mMinimumSampleIntervalMs == canFrame.minimumSampleIntervalMs ) ) { found = true; - buf.mSize = std::max( buf.mSize, c.sampleBufferSize ); + buf.mSize = std::max( buf.mSize, canFrame.sampleBufferSize ); break; } } if ( !found ) { - mCanFrameBuffers.emplace_back( c.frameID, c.channelID, c.sampleBufferSize, c.minimumSampleIntervalMs ); + mCanFrameBuffers.emplace_back( + canFrame.frameID, canFrame.channelID, canFrame.sampleBufferSize, canFrame.minimumSampleIntervalMs ); } } } - // At this point all buffers should be resized to correct size. Now pointer to std::vector elements can be used - for ( size_t conditionIndex = 0; conditionIndex < mConditions.size(); conditionIndex++ ) + for ( uint32_t conditionIndex = 0; conditionIndex < mConditions.size(); conditionIndex++ ) { - auto &ac = mConditions[conditionIndex]; - for ( auto &s : ac.mCondition.signals ) + auto &activeCondition = mConditions[conditionIndex]; + + if ( activeCondition.mCondition.isStaticCondition ) { - switch ( s.signalType ) + // evaluate static condition once when inspection matrix is handed over + evaluateStaticCondition( conditionIndex ); + } + + for ( auto &signal : activeCondition.mCondition.signals ) + { + switch ( signal.signalType ) { case SignalType::UINT8: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::INT8: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::UINT16: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::INT16: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::UINT32: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::INT32: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::UINT64: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::INT64: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::FLOAT: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::DOUBLE: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; case SignalType::BOOLEAN: - updateConditionBuffer( s, ac, conditionIndex ); + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; - default: + case SignalType::UNKNOWN: + // Signal of type UNKNOWN should not be processed + break; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + updateConditionBuffer( signal, activeCondition, conditionIndex ); break; +#endif } } // Overwrite last trigger time 0 with current time to avoid trigger at time 0 - ac.mLastTrigger = currentTime; - } + activeCondition.mLastTrigger = currentTime; - // Assume all conditions are currently true; - mConditionsWithConditionCurrentlyTrue.set(); + for ( const auto &signal : activeCondition.mCondition.signals ) + { + if ( !signal.isConditionOnlySignal ) + { + activeCondition.mCollectedSignalIds.emplace( signal.signalID ); + } + } + } - (void)preAllocateBuffers(); + static_cast( preAllocateBuffers() ); } template void CollectionInspectionEngine::updateConditionBuffer( - const InspectionMatrixSignalCollectionInfo &inspectionMatrixCollectionInfoIn, - ActiveCondition &acIn, - const long unsigned int conditionIndexIn ) + const InspectionMatrixSignalCollectionInfo &inspectionMatrixCollectionInfo, + ActiveCondition &activeCondition, + const long unsigned int conditionIndex ) { - SignalID signalIDIn = inspectionMatrixCollectionInfoIn.signalID; - SignalHistoryBuffer *buf = nullptr; - std::vector> *signalHistoryBufferPtr = nullptr; - - signalHistoryBufferPtr = getSignalHistoryBufferPtr( signalIDIn ); - - if ( signalHistoryBufferPtr != nullptr ) - { - auto &bufferVec = *signalHistoryBufferPtr; - for ( auto &buffer : bufferVec ) - { - if ( buffer.mMinimumSampleIntervalMs == inspectionMatrixCollectionInfoIn.minimumSampleIntervalMs ) - { - buf = &buffer; - break; - } - } - } - if ( ( buf != nullptr ) && isSignalPartOfEval( acIn.mCondition.condition, signalIDIn, MAX_EQUATION_DEPTH ) ) + SignalID signalID = inspectionMatrixCollectionInfo.signalID; + auto buf = getSignalHistoryBufferPtr( signalID, inspectionMatrixCollectionInfo.minimumSampleIntervalMs ); + if ( buf != nullptr ) { - buf->mConditionsThatEvaluateOnThisSignal.set( conditionIndexIn ); - // acIn.mEvaluationSignals[signalIDIn] = buf; - acIn.mEvaluationSignals.insert( { signalIDIn, buf } ); + buf->mConditionsThatEvaluateOnThisSignal.set( conditionIndex ); + activeCondition.mConditionSignals.insert( { signalID, buf } ); FixedTimeWindowFunctionData *window = - buf->getFixedWindow( inspectionMatrixCollectionInfoIn.fixedWindowPeriod ); + buf->getFixedWindow( inspectionMatrixCollectionInfo.fixedWindowPeriod ); if ( window != nullptr ) { - // acIn.mEvaluationFunctions[signalIDIn] = window; - acIn.mEvaluationFunctions.insert( { signalIDIn, window } ); + // activeCondition.mEvaluationFunctions[signalID] = window; + activeCondition.mEvaluationFunctions.insert( { signalID, window } ); } } } template bool -CollectionInspectionEngine::allocateBufferVector( SignalID signalIDIn, uint32_t &usedBytes ) +CollectionInspectionEngine::allocateBufferVector( SignalID signalID, size_t &usedBytes ) { - std::vector> *signalHistoryBufferPtr = nullptr; - signalHistoryBufferPtr = getSignalHistoryBufferPtr( signalIDIn ); - - if ( signalHistoryBufferPtr != nullptr ) + auto signalHistoryBufferVectorPtr = getSignalHistoryBuffersPtr( signalID ); + if ( signalHistoryBufferVectorPtr != nullptr ) { - auto &bufferVec = *signalHistoryBufferPtr; - for ( auto &signal : bufferVec ) + auto &signalHistoryBufferVector = *signalHistoryBufferVectorPtr; + for ( auto &buffer : signalHistoryBufferVector ) { - uint64_t requiredBytes = signal.mSize * static_cast( sizeof( struct SignalSample ) ); + uint64_t requiredBytes = buffer.mSize * static_cast( sizeof( struct SignalSample ) ); if ( usedBytes + requiredBytes > MAX_SAMPLE_MEMORY ) { - FWE_LOG_WARN( "The requested " + std::to_string( signal.mSize ) + + FWE_LOG_WARN( "The requested " + std::to_string( buffer.mSize ) + " number of signal samples leads to a memory requirement that's above the maximum " "configured of " + std::to_string( MAX_SAMPLE_MEMORY ) + "Bytes" ); - signal.mSize = 0; + buffer.mSize = 0; TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::COLLECTION_SCHEME_ERROR ); return false; } - usedBytes += static_cast( requiredBytes ); + usedBytes += static_cast( requiredBytes ); // reserve the size like new[] - signal.mBuffer.resize( signal.mSize ); + buffer.mBuffer.resize( buffer.mSize ); } } return true; @@ -310,7 +300,7 @@ bool CollectionInspectionEngine::preAllocateBuffers() { // Allocate size - uint32_t usedBytes = 0; + size_t usedBytes = 0; // Allocate Signal Buffer for ( auto &bufferVector : mSignalBuffers ) @@ -388,7 +378,7 @@ CollectionInspectionEngine::preAllocateBuffers() } break; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: + case SignalType::COMPLEX_SIGNAL: if ( !allocateBufferVector( signalID, usedBytes ) ) { return false; @@ -404,7 +394,7 @@ CollectionInspectionEngine::preAllocateBuffers() // Allocate Can buffer for ( auto &buf : mCanFrameBuffers ) { - uint64_t requiredBytes = buf.mSize * static_cast( sizeof( struct CanFrameSample ) ); + size_t requiredBytes = buf.mSize * sizeof( struct CanFrameSample ); if ( usedBytes + requiredBytes > MAX_SAMPLE_MEMORY ) { FWE_LOG_WARN( "The requested " + std::to_string( buf.mSize ) + @@ -414,7 +404,7 @@ CollectionInspectionEngine::preAllocateBuffers() buf.mSize = 0; return false; } - usedBytes += static_cast( requiredBytes ); + usedBytes += requiredBytes; // reserve the size like new[] buf.mBuffer.resize( buf.mSize ); @@ -434,32 +424,30 @@ CollectionInspectionEngine::clear() mNextWindowFunctionTimesOut = 0; mConditionsWithInputSignalChanged.reset(); mConditionsWithConditionCurrentlyTrue.reset(); - mConditionsNotTriggeredWaitingPublished.reset(); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + mConditionsTriggeredWaitingPublished.reset(); if ( mRawBufferManager != nullptr ) { mRawBufferManager->resetUsageHintsForStage( RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_HISTORY_BUFFER ); } -#endif } template void -CollectionInspectionEngine::updateBufferFixedWindowFunction( SignalID signalIDIn, InspectionTimestamp timestamp ) +CollectionInspectionEngine::updateBufferFixedWindowFunction( SignalID signalID, Timestamp timestamp ) { std::vector> *signalHistoryBufferPtr = nullptr; try { - if ( mSignalBuffers.find( signalIDIn ) != mSignalBuffers.end() ) + if ( mSignalBuffers.find( signalID ) != mSignalBuffers.end() ) { - auto &mapVal = mSignalBuffers.at( signalIDIn ); + auto &mapVal = mSignalBuffers.at( signalID ); signalHistoryBufferPtr = boost::get>>( &mapVal ); } } catch ( ... ) { - FWE_LOG_ERROR( "Failed to retrieve signalHistoryBuffer vector for signal ID " + std::to_string( signalIDIn ) ); + FWE_LOG_ERROR( "Failed to retrieve signalHistoryBuffer vector for signal ID " + std::to_string( signalID ) ); return; } if ( signalHistoryBufferPtr != nullptr ) @@ -480,16 +468,17 @@ CollectionInspectionEngine::updateBufferFixedWindowFunction( SignalID signalIDIn } void -CollectionInspectionEngine::updateAllFixedWindowFunctions( InspectionTimestamp timestamp ) +CollectionInspectionEngine::updateAllFixedWindowFunctions( Timestamp timestamp ) { - mNextWindowFunctionTimesOut = std::numeric_limits::max(); + mNextWindowFunctionTimesOut = std::numeric_limits::max(); for ( auto &signalVector : mSignalBuffers ) { auto signalID = signalVector.first; if ( mSignalToBufferTypeMap.find( signalID ) != mSignalToBufferTypeMap.end() ) { - auto signalType = mSignalToBufferTypeMap[signalID]; + // coverity[autosar_cpp14_m6_4_6_violation] + // coverity[misra_cpp_2008_rule_6_4_6_violation] compiler warning is preferred over a default-clause switch ( signalType ) { case SignalType::UINT8: @@ -511,7 +500,6 @@ CollectionInspectionEngine::updateAllFixedWindowFunctions( InspectionTimestamp t updateBufferFixedWindowFunction( signalID, timestamp ); break; case SignalType::UINT64: - updateBufferFixedWindowFunction( signalID, timestamp ); break; case SignalType::INT64: @@ -526,127 +514,144 @@ CollectionInspectionEngine::updateAllFixedWindowFunctions( InspectionTimestamp t case SignalType::BOOLEAN: updateBufferFixedWindowFunction( signalID, timestamp ); break; - default: + case SignalType::UNKNOWN: + FWE_LOG_WARN( "Window functions are not supported for signal ID: " + std::to_string( signalID ) + + " as it is of type UNKNOWN" ); + break; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + // Window functions are not supported for complex signals break; +#endif } } } } +void +CollectionInspectionEngine::evaluateStaticCondition( uint32_t conditionIndex ) +{ + ActiveCondition &condition = mConditions[conditionIndex]; + InspectionValue result; + ExpressionErrorCode ret = + eval( condition.mCondition.condition, condition, result, MAX_EQUATION_DEPTH, conditionIndex ); + if ( ( ret != ExpressionErrorCode::SUCCESSFUL ) || ( result.type != InspectionValue::DataType::BOOL ) || + ( !result.boolVal ) ) + { + // Flip default true flag to false if static condition is evaluated to false + mConditionsWithConditionCurrentlyTrue.reset( conditionIndex ); + } +} + bool CollectionInspectionEngine::evaluateConditions( const TimePoint ¤tTime ) { - bool oneConditionIsTrue = false; + bool oneConditionEvaluatedToTrue = false; + // if any sampling window times out there is a new value available to be processed by a condition if ( currentTime.monotonicTimeMs >= mNextWindowFunctionTimesOut ) { updateAllFixedWindowFunctions( currentTime.monotonicTimeMs ); } - auto conditionsToEvaluate = ( mConditionsWithConditionCurrentlyTrue | mConditionsWithInputSignalChanged ) & - mConditionsNotTriggeredWaitingPublished; - if ( conditionsToEvaluate.none() ) - { - // No conditions to evaluate - return false; - } + // faster implementation like find next bit set to one would be possible but for example // conditionsToEvaluate._Find_first is not part of C++ standard for ( uint32_t i = 0; i < mConditions.size(); i++ ) { - if ( conditionsToEvaluate.test( i ) ) + ActiveCondition &condition = mConditions[i]; + bool conditionEvaluatedToTrue = false; + // Only reevaluate non-static conditions with changed input + if ( ( ( mConditionsWithInputSignalChanged.test( i ) ) && ( !condition.mCondition.isStaticCondition ) ) || + ( condition.mCondition.alwaysEvaluateCondition ) ) { - ActiveCondition &condition = mConditions[i]; - InspectionValue result = 0; - bool resultBool = false; - mConditionsWithInputSignalChanged.reset( i ); - ExpressionErrorCode ret = - eval( condition.mCondition.condition, condition, result, resultBool, MAX_EQUATION_DEPTH ); - if ( ( ret == ExpressionErrorCode::SUCCESSFUL ) && resultBool ) + InspectionValue result; + ExpressionErrorCode ret = eval( condition.mCondition.condition, condition, result, MAX_EQUATION_DEPTH, i ); + if ( ( ret != ExpressionErrorCode::SUCCESSFUL ) || ( !result.isBoolOrDouble() ) || ( !result.asBool() ) ) { - // Only evaluate condition to true if minimumPublishIntervalMs has passed - if ( currentTime.monotonicTimeMs >= - condition.mLastTrigger.monotonicTimeMs + condition.mCondition.minimumPublishIntervalMs ) - { - if ( ( !condition.mCondition.triggerOnlyOnRisingEdge ) || - ( !mConditionsWithConditionCurrentlyTrue.test( i ) ) ) - { - // Mark condition for the upload - mConditionsNotTriggeredWaitingPublished.reset( i ); - condition.mLastTrigger = currentTime; - } - mConditionsWithConditionCurrentlyTrue.set( i ); - oneConditionIsTrue = true; - } + mConditionsWithConditionCurrentlyTrue.reset( i ); } else { - mConditionsWithConditionCurrentlyTrue.reset( i ); + conditionEvaluatedToTrue = true; + } + mConditionsWithInputSignalChanged.reset( i ); + } + // If condition was reevaluated to true or if condition is still true and not waiting to be published + // Check if condition can be "retriggered" and marked for the upload + if ( ( conditionEvaluatedToTrue ) || ( ( mConditionsWithConditionCurrentlyTrue.test( i ) ) && + ( !mConditionsTriggeredWaitingPublished.test( i ) ) ) ) + { + // Mark conditions for upload only if minimumPublishIntervalMs has passed + if ( currentTime.monotonicTimeMs >= + condition.mLastTrigger.monotonicTimeMs + condition.mCondition.minimumPublishIntervalMs ) + { + if ( ( ( !condition.mCondition.triggerOnlyOnRisingEdge ) || + ( !mConditionsWithConditionCurrentlyTrue.test( i ) ) ) && + ( !mConditionsTriggeredWaitingPublished.test( i ) ) ) + { + // Mark condition for the upload + mConditionsTriggeredWaitingPublished.set( i ); + condition.mLastTrigger = currentTime; + // Prepare the collected data, but don't fill it from the signal history buffers yet + condition.mCollectedData = { std::make_shared(), +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + }; + } + mConditionsWithConditionCurrentlyTrue.set( i ); + oneConditionEvaluatedToTrue = true; } } } - return oneConditionIsTrue; + return oneConditionEvaluatedToTrue; } -template +template void -CollectionInspectionEngine::collectLastSignals( InspectionSignalID id, - uint32_t minimumSamplingInterval, - uint32_t maxNumberOfSignalsToCollect, +CollectionInspectionEngine::collectLastSignals( SignalID id, + size_t maxNumberOfSignalsToCollect, uint32_t conditionId, - SignalType signalTypeIn, - InspectionTimestamp &newestSignalTimestamp, + SignalType signalType, + Timestamp &newestSignalTimestamp, std::vector &output ) { - if ( mSignalBuffers[id].empty() ) + auto buf = mConditions[conditionId].getEvaluationSignalsBufferPtr( id ); + if ( buf == nullptr ) { - // Signal not collected by any active condition + // Signal not collected by any active condition or access by Invalid DataType return; } - std::vector> *signalHistoryBufferPtr = nullptr; - signalHistoryBufferPtr = getSignalHistoryBufferPtr( id ); - if ( signalHistoryBufferPtr == nullptr ) + int pos = static_cast( buf->mCurrentPosition ); + for ( size_t i = 0; i < std::min( maxNumberOfSignalsToCollect, buf->mCounter ); i++ ) { - // Access by Invalid DataType - return; - } - auto &bufferVec = *signalHistoryBufferPtr; - for ( auto &buf : bufferVec ) - { - if ( ( buf.mMinimumSampleIntervalMs == minimumSamplingInterval ) && ( buf.mSize > 0 ) ) + // Ensure access is in bounds + if ( pos < 0 ) { - int pos = static_cast( buf.mCurrentPosition ); - for ( uint32_t i = 0; i < std::min( maxNumberOfSignalsToCollect, buf.mCounter ); i++ ) - { - // Ensure access is in bounds - if ( pos < 0 ) - { - pos = static_cast( buf.mSize ) - 1; - } - if ( pos >= static_cast( buf.mSize ) ) - { - pos = 0; - } - auto &sample = buf.mBuffer[static_cast( pos )]; - if ( ( !sample.isAlreadyConsumed( conditionId ) ) || ( !mSendDataOnlyOncePerCondition ) ) - { - output.emplace_back( id, sample.mTimestamp, sample.mValue, signalTypeIn ); - sample.setAlreadyConsumed( conditionId, true ); + pos = static_cast( buf->mSize ) - 1; + } + if ( pos >= static_cast( buf->mSize ) ) + { + pos = 0; + } + auto &sample = buf->mBuffer[static_cast( pos )]; + if ( ( !sample.isAlreadyConsumed( conditionId ) ) || ( !mSendDataOnlyOncePerCondition ) ) + { + output.emplace_back( id, sample.mTimestamp, sample.mValue, signalType ); + sample.setAlreadyConsumed( conditionId, true ); #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - if ( signalTypeIn == SignalType::RAW_DATA_BUFFER_HANDLE ) - { - NotifyRawBufferManager::increaseElementUsage( - id, - mRawBufferManager.get(), - RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_SELECTED_FOR_UPLOAD, - sample.mValue ); - } -#endif - } - newestSignalTimestamp = std::max( newestSignalTimestamp, sample.mTimestamp ); - pos--; + if ( signalType == SignalType::COMPLEX_SIGNAL ) + { + NotifyRawBufferManager::increaseElementUsage( + id, + mRawBufferManager.get(), + RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_SELECTED_FOR_UPLOAD, + sample.mValue ); } - return; +#endif } + newestSignalTimestamp = std::max( newestSignalTimestamp, sample.mTimestamp ); + pos--; } } @@ -654,9 +659,9 @@ void CollectionInspectionEngine::collectLastCanFrames( CANRawFrameID canID, CANChannelNumericID channelID, uint32_t minimumSamplingInterval, - uint32_t maxNumberOfSignalsToCollect, + size_t maxNumberOfSignalsToCollect, uint32_t conditionId, - InspectionTimestamp &newestSignalTimestamp, + Timestamp &newestSignalTimestamp, std::vector &output ) { for ( auto &buf : mCanFrameBuffers ) @@ -690,14 +695,20 @@ CollectionInspectionEngine::collectLastCanFrames( CANRawFrameID canID, } } -std::shared_ptr +void CollectionInspectionEngine::collectData( ActiveCondition &condition, uint32_t conditionId, - InspectionTimestamp &newestSignalTimestamp ) + Timestamp &newestSignalTimestamp, + CollectionInspectionEngineOutput &output ) { - std::shared_ptr collectedData = std::make_shared(); - collectedData->metadata = condition.mCondition.metadata; - collectedData->triggerTime = condition.mLastTrigger.systemTimeMs; + output.triggeredCollectionSchemeData->metadata = condition.mCondition.metadata; + output.triggeredCollectionSchemeData->triggerTime = condition.mLastTrigger.systemTimeMs; + output.triggeredCollectionSchemeData->eventID = condition.mEventID; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + output.triggeredVisionSystemData->metadata = condition.mCondition.metadata; + output.triggeredVisionSystemData->triggerTime = condition.mLastTrigger.systemTimeMs; + output.triggeredVisionSystemData->eventID = condition.mEventID; +#endif // Pack signals for ( auto &s : condition.mCondition.signals ) { @@ -707,114 +718,107 @@ CollectionInspectionEngine::collectData( ActiveCondition &condition, { case SignalType::UINT8: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::INT8: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::UINT16: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::INT16: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::UINT32: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::INT32: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::UINT64: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::INT64: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::FLOAT: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::DOUBLE: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); break; case SignalType::BOOLEAN: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); + output.triggeredCollectionSchemeData->signals ); + break; + case SignalType::UNKNOWN: + FWE_LOG_WARN( "Signal ID: " + std::to_string( s.signalID ) + " associated with Campaign SyncId: " + + ( condition.mCondition.metadata.collectionSchemeID ) + + " is of UNKNOWN type and will not be collected" ); break; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: + case SignalType::COMPLEX_SIGNAL: collectLastSignals( s.signalID, - s.minimumSampleIntervalMs, s.sampleBufferSize, conditionId, s.signalType, newestSignalTimestamp, - collectedData->signals ); -#endif + output.triggeredVisionSystemData->signals ); break; +#endif } } } @@ -828,28 +832,32 @@ CollectionInspectionEngine::collectData( ActiveCondition &condition, c.sampleBufferSize, conditionId, newestSignalTimestamp, - collectedData->canFrames ); + output.triggeredCollectionSchemeData->canFrames ); } // Pack active DTCs if any if ( condition.mCondition.includeActiveDtcs && ( ( !isActiveDTCsConsumed( conditionId ) ) || mSendDataOnlyOncePerCondition ) ) { - collectedData->mDTCInfo = mActiveDTCs; + output.triggeredCollectionSchemeData->mDTCInfo = mActiveDTCs; setActiveDTCsConsumed( conditionId, true ); } - // Propagate the event ID - collectedData->eventID = condition.mEventID; - return std::const_pointer_cast( collectedData ); + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + if ( output.triggeredVisionSystemData->signals.empty() ) + { + output.triggeredVisionSystemData = nullptr; + } +#endif } -std::shared_ptr +CollectionInspectionEngineOutput CollectionInspectionEngine::collectNextDataToSend( const TimePoint ¤tTime, uint32_t &waitTimeMs ) { uint32_t minimumWaitTimeMs = std::numeric_limits::max(); - if ( mConditionsNotTriggeredWaitingPublished.all() ) + if ( mConditionsTriggeredWaitingPublished.none() ) { waitTimeMs = minimumWaitTimeMs; - return std::shared_ptr( nullptr ); + return {}; } for ( uint32_t i = 0; i < mConditions.size(); i++ ) { @@ -857,41 +865,46 @@ CollectionInspectionEngine::collectNextDataToSend( const TimePoint ¤tTime, { mNextConditionToCollectedIndex = 0; } - if ( !mConditionsNotTriggeredWaitingPublished.test( mNextConditionToCollectedIndex ) ) + if ( mConditionsTriggeredWaitingPublished.test( mNextConditionToCollectedIndex ) ) { auto &condition = mConditions[mNextConditionToCollectedIndex]; - - if ( ( ( condition.mLastTrigger.systemTimeMs == 0 ) && ( condition.mLastTrigger.monotonicTimeMs == 0 ) ) || - ( currentTime.monotonicTimeMs >= - condition.mLastTrigger.monotonicTimeMs + condition.mCondition.afterDuration ) ) { - mConditionsNotTriggeredWaitingPublished.set( mNextConditionToCollectedIndex ); - // Generate the Event ID and pack it into the active Condition - condition.mEventID = generateEventID( currentTime.systemTimeMs ); - // Return the collected data - InspectionTimestamp newestSignalTimeStamp = 0; - auto cd = collectData( condition, mNextConditionToCollectedIndex, newestSignalTimeStamp ); - // After collecting the data set the newest timestamp from any data that was - // collected - condition.mLastDataTimestampPublished = std::min( newestSignalTimeStamp, currentTime.monotonicTimeMs ); - // Increase index before returning from the function - mNextConditionToCollectedIndex++; - return cd; - } - else - { - minimumWaitTimeMs = - std::min( minimumWaitTimeMs, - static_cast( ( condition.mLastTrigger.monotonicTimeMs + - condition.mCondition.afterDuration ) - - currentTime.monotonicTimeMs ) ); + if ( ( ( condition.mLastTrigger.systemTimeMs == 0 ) && + ( condition.mLastTrigger.monotonicTimeMs == 0 ) ) || + ( currentTime.monotonicTimeMs >= + condition.mLastTrigger.monotonicTimeMs + condition.mCondition.afterDuration ) ) + { + // Mark as not triggered since data is going to be collected + mConditionsTriggeredWaitingPublished.reset( mNextConditionToCollectedIndex ); + // Generate the Event ID and pack it into the active Condition + condition.mEventID = generateEventID( currentTime.systemTimeMs ); + // Return the collected data + Timestamp newestSignalTimeStamp = 0; + collectData( + condition, mNextConditionToCollectedIndex, newestSignalTimeStamp, condition.mCollectedData ); + // After collecting the data set the newest timestamp from any data that was + // collected + condition.mLastDataTimestampPublished = + std::min( newestSignalTimeStamp, currentTime.monotonicTimeMs ); + // Increase index before returning from the function + mNextConditionToCollectedIndex++; + return std::move( condition.mCollectedData ); + } + else + { + minimumWaitTimeMs = + std::min( minimumWaitTimeMs, + static_cast( ( condition.mLastTrigger.monotonicTimeMs + + condition.mCondition.afterDuration ) - + currentTime.monotonicTimeMs ) ); + } } } mNextConditionToCollectedIndex++; } // No Data ready to be sent waitTimeMs = minimumWaitTimeMs; - return std::shared_ptr( nullptr ); + return {}; } void @@ -938,8 +951,8 @@ CollectionInspectionEngine::setActiveDTCs( const DTCInfo &activeDTCs ) } template -CollectionInspectionEngine::ExpressionErrorCode -CollectionInspectionEngine::getLatestBufferSignalValue( InspectionSignalID id, +ExpressionErrorCode +CollectionInspectionEngine::getLatestBufferSignalValue( SignalID id, ActiveCondition &condition, InspectionValue &result ) { @@ -960,10 +973,8 @@ CollectionInspectionEngine::getLatestBufferSignalValue( InspectionSignalID id, return ExpressionErrorCode::SUCCESSFUL; } -CollectionInspectionEngine::ExpressionErrorCode -CollectionInspectionEngine::getLatestSignalValue( InspectionSignalID id, - ActiveCondition &condition, - InspectionValue &result ) +ExpressionErrorCode +CollectionInspectionEngine::getLatestSignalValue( SignalID id, ActiveCondition &condition, InspectionValue &result ) { if ( mSignalToBufferTypeMap.find( id ) == mSignalToBufferTypeMap.end() ) { @@ -972,51 +983,49 @@ CollectionInspectionEngine::getLatestSignalValue( InspectionSignalID id, return ExpressionErrorCode::SIGNAL_NOT_FOUND; } auto signalType = mSignalToBufferTypeMap[id]; + // coverity[autosar_cpp14_m6_4_6_violation] + // coverity[misra_cpp_2008_rule_6_4_6_violation] compiler warning is preferred over a default-clause switch ( signalType ) { case SignalType::UINT8: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::INT8: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::UINT16: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::INT16: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::UINT32: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::INT32: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::UINT64: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::INT64: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::FLOAT: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::DOUBLE: return getLatestBufferSignalValue( id, condition, result ); - break; case SignalType::BOOLEAN: return getLatestBufferSignalValue( id, condition, result ); - break; - default: + case SignalType::UNKNOWN: + FWE_LOG_WARN( "Signal ID: " + std::to_string( id ) + " associated with Campaign SyncId: " + + condition.mCondition.metadata.collectionSchemeID + " is of type UNKNOWN and used in evaluation" ); + return ExpressionErrorCode::SIGNAL_NOT_FOUND; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + FWE_LOG_WARN( "Complex signals are not supported in evaluation" ) return ExpressionErrorCode::SIGNAL_NOT_FOUND; - break; +#endif } + return ExpressionErrorCode::SIGNAL_NOT_FOUND; } -template -CollectionInspectionEngine::ExpressionErrorCode +template +ExpressionErrorCode CollectionInspectionEngine::getSampleWindowFunctionType( WindowFunction function, - InspectionSignalID signalID, + SignalID signalID, ActiveCondition &condition, InspectionValue &result ) { @@ -1055,9 +1064,9 @@ CollectionInspectionEngine::getSampleWindowFunctionType( WindowFunction function } } -CollectionInspectionEngine::ExpressionErrorCode +ExpressionErrorCode CollectionInspectionEngine::getSampleWindowFunction( WindowFunction function, - InspectionSignalID signalID, + SignalID signalID, ActiveCondition &condition, InspectionValue &result ) { @@ -1068,53 +1077,52 @@ CollectionInspectionEngine::getSampleWindowFunction( WindowFunction function, return ExpressionErrorCode::SIGNAL_NOT_FOUND; } auto signalType = mSignalToBufferTypeMap[signalID]; + // coverity[autosar_cpp14_m6_4_6_violation] + // coverity[misra_cpp_2008_rule_6_4_6_violation] compiler warning is preferred over a default-clause switch ( signalType ) { case SignalType::UINT8: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::INT8: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::UINT16: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::INT16: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::UINT32: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::INT32: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::UINT64: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::INT64: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::FLOAT: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::DOUBLE: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; case SignalType::BOOLEAN: return getSampleWindowFunctionType( function, signalID, condition, result ); - break; - default: + case SignalType::UNKNOWN: + FWE_LOG_WARN( "Window functions are not supported for signal ID: " + std::to_string( signalID ) + + " associated with Campaign SyncId: " + condition.mCondition.metadata.collectionSchemeID + + " as signal is of type UNKNOWN" ); + return ExpressionErrorCode::SIGNAL_NOT_FOUND; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + FWE_LOG_WARN( "Window functions are not supported for complex signals" ) return ExpressionErrorCode::SIGNAL_NOT_FOUND; - break; +#endif } + return ExpressionErrorCode::SIGNAL_NOT_FOUND; } -CollectionInspectionEngine::ExpressionErrorCode +ExpressionErrorCode CollectionInspectionEngine::eval( const ExpressionNode *expression, ActiveCondition &condition, - InspectionValue &resultValueDouble, - bool &resultValueBool, - int remainingStackDepth ) + InspectionValue &resultValue, + int remainingStackDepth, + uint32_t conditionId ) { if ( ( remainingStackDepth <= 0 ) || ( expression == nullptr ) ) { @@ -1123,32 +1131,30 @@ CollectionInspectionEngine::eval( const ExpressionNode *expression, } if ( expression->nodeType == ExpressionNodeType::FLOAT ) { - resultValueDouble = expression->floatingValue; + resultValue = expression->floatingValue; return ExpressionErrorCode::SUCCESSFUL; } if ( expression->nodeType == ExpressionNodeType::BOOLEAN ) { - resultValueBool = expression->booleanValue; + resultValue = expression->booleanValue; return ExpressionErrorCode::SUCCESSFUL; } if ( expression->nodeType == ExpressionNodeType::SIGNAL ) { - return getLatestSignalValue( expression->signalID, condition, resultValueDouble ); + return getLatestSignalValue( expression->signalID, condition, resultValue ); } - if ( expression->nodeType == ExpressionNodeType::WINDOWFUNCTION ) + if ( expression->nodeType == ExpressionNodeType::WINDOW_FUNCTION ) { return getSampleWindowFunction( - expression->function.windowFunction, expression->signalID, condition, resultValueDouble ); + expression->function.windowFunction, expression->signalID, condition, resultValue ); } - InspectionValue leftDouble = 0; - InspectionValue rightDouble = 0; - bool leftBool = false; - bool rightBool = false; + InspectionValue leftResult; + InspectionValue rightResult; ExpressionErrorCode leftRet = ExpressionErrorCode::SUCCESSFUL; ExpressionErrorCode rightRet = ExpressionErrorCode::SUCCESSFUL; // Recursion limited depth through last parameter - leftRet = eval( expression->left, condition, leftDouble, leftBool, remainingStackDepth - 1 ); + leftRet = eval( expression->left, condition, leftResult, remainingStackDepth - 1, conditionId ); if ( leftRet != ExpressionErrorCode::SUCCESSFUL ) { @@ -1159,7 +1165,7 @@ CollectionInspectionEngine::eval( const ExpressionNode *expression, if ( expression->nodeType != ExpressionNodeType::OPERATOR_LOGICAL_NOT ) { // No short-circuit evaluation so always evaluate right part - rightRet = eval( expression->right, condition, rightDouble, rightBool, remainingStackDepth - 1 ); + rightRet = eval( expression->right, condition, rightResult, remainingStackDepth - 1, conditionId ); if ( rightRet != ExpressionErrorCode::SUCCESSFUL ) { @@ -1170,43 +1176,104 @@ CollectionInspectionEngine::eval( const ExpressionNode *expression, switch ( expression->nodeType ) { case ExpressionNodeType::OPERATOR_SMALLER: - resultValueBool = leftDouble < rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() < rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_BIGGER: - resultValueBool = leftDouble > rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() > rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_SMALLER_EQUAL: - resultValueBool = leftDouble <= rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() <= rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_BIGGER_EQUAL: - resultValueBool = leftDouble >= rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() >= rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_EQUAL: - resultValueBool = std::abs( leftDouble - rightDouble ) < EVAL_EQUAL_DISTANCE(); + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + else + { + resultValue = std::abs( leftResult.asDouble() - rightResult.asDouble() ) < EVAL_EQUAL_DISTANCE(); + } return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_NOT_EQUAL: - resultValueBool = !( std::abs( leftDouble - rightDouble ) < EVAL_EQUAL_DISTANCE() ); + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + else + { + resultValue = !( std::abs( leftResult.asDouble() - rightResult.asDouble() ) < EVAL_EQUAL_DISTANCE() ); + } return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_LOGICAL_AND: - resultValueBool = leftBool && rightBool; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asBool() && rightResult.asBool(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_LOGICAL_OR: - resultValueBool = leftBool || rightBool; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asBool() || rightResult.asBool(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_LOGICAL_NOT: - resultValueBool = !leftBool; + if ( !leftResult.isBoolOrDouble() ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = !leftResult.asBool(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_ARITHMETIC_PLUS: - resultValueDouble = leftDouble + rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + else + { + resultValue = leftResult.asDouble() + rightResult.asDouble(); + } return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_ARITHMETIC_MINUS: - resultValueDouble = leftDouble - rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() - rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_ARITHMETIC_MULTIPLY: - resultValueDouble = leftDouble * rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() * rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; case ExpressionNodeType::OPERATOR_ARITHMETIC_DIVIDE: - resultValueDouble = leftDouble / rightDouble; + if ( ( !leftResult.isBoolOrDouble() ) || ( !rightResult.isBoolOrDouble() ) ) + { + return ExpressionErrorCode::TYPE_MISMATCH; + } + resultValue = leftResult.asDouble() / rightResult.asDouble(); return ExpressionErrorCode::SUCCESSFUL; default: return ExpressionErrorCode::NOT_IMPLEMENTED_TYPE; @@ -1214,7 +1281,7 @@ CollectionInspectionEngine::eval( const ExpressionNode *expression, } EventID -CollectionInspectionEngine::generateEventID( InspectionTimestamp timestamp ) +CollectionInspectionEngine::generateEventID( Timestamp timestamp ) { // Generate an eventId as a combination of an event counter and a timestamp uint32_t eventId = static_cast( generateEventCounter() ) | static_cast( timestamp << 8 ); diff --git a/src/CollectionInspectionEngine.h b/src/CollectionInspectionEngine.h index 7fefd69a..d0b1613e 100644 --- a/src/CollectionInspectionEngine.h +++ b/src/CollectionInspectionEngine.h @@ -10,40 +10,38 @@ #include "LoggingModule.h" #include "MessageTypes.h" #include "OBDDataTypes.h" +#include "RawDataManager.h" #include "SignalTypes.h" #include "TimeTypes.h" #include #include #include #include // As _Find_first() is not part of C++ standard and compiler specific other structure could be considered +#include #include +#include #include #include #include #include +#include #include +#include #include #include -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "RawDataManager.h" -#endif - namespace Aws { namespace IoTFleetWise { -using InspectionSignalID = uint32_t; - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA // Rule A14-8-2 suggests to use class template specialization instead of function template specialization template class NotifyRawBufferManager { public: static bool - increaseElementUsage( InspectionSignalID id, + increaseElementUsage( SignalID id, RawData::BufferManager *rawBufferManager, RawData::BufferHandleUsageStage stage, T value ) @@ -57,7 +55,7 @@ class NotifyRawBufferManager } static bool - decreaseElementUsage( InspectionSignalID id, + decreaseElementUsage( SignalID id, RawData::BufferManager *rawBufferManager, RawData::BufferHandleUsageStage stage, T value ) @@ -76,7 +74,7 @@ class NotifyRawBufferManager { public: static bool - increaseElementUsage( InspectionSignalID id, + increaseElementUsage( SignalID id, RawData::BufferManager *rawBufferManager, RawData::BufferHandleUsageStage stage, RawData::BufferHandle value ) @@ -90,7 +88,7 @@ class NotifyRawBufferManager } static bool - decreaseElementUsage( InspectionSignalID id, + decreaseElementUsage( SignalID id, RawData::BufferManager *rawBufferManager, RawData::BufferHandleUsageStage stage, RawData::BufferHandle value ) @@ -103,7 +101,6 @@ class NotifyRawBufferManager return false; } }; -#endif /** * @brief Main class to implement collection and inspection engine logic @@ -116,9 +113,6 @@ class CollectionInspectionEngine { public: - using InspectionTimestamp = uint64_t; - using InspectionValue = double; - /** * @brief Construct the CollectionInspectionEngine which handles multiple conditions * @@ -155,8 +149,7 @@ class CollectionInspectionEngine * * @return if dataReadyToBeSent() is false a nullptr otherwise the collected data will be returned */ - std::shared_ptr collectNextDataToSend( const TimePoint ¤tTime, - uint32_t &waitTimeMs ); + CollectionInspectionEngineOutput collectNextDataToSend( const TimePoint ¤tTime, uint32_t &waitTimeMs ); /** * @brief Give a new signal to the collection engine to cached it * @@ -168,13 +161,20 @@ class CollectionInspectionEngine * * @param id id of the obd based or can based signal * @param receiveTime timestamp at which time was the signal seen on the physical bus + * @param currentMonotonicTimeMs current monotonic time for window function evaluation * @param value the signal value */ template - void addNewSignal( InspectionSignalID id, const TimePoint &receiveTime, T value ); + void addNewSignal( SignalID id, const TimePoint &receiveTime, const Timestamp ¤tMonotonicTimeMs, T value ); - template - void addSignalToBuffer( const InspectionMatrixSignalCollectionInfo &signalIn ); + /** + * @brief Add new signal buffer entry to mSignalBuffers (SignalHistoryBufferCollection) + * + * There is one buffer per sampling interval. For each sampling interval buffer us resized to the biggest requested + * size. + */ + template + void addSignalBuffer( const InspectionMatrixSignalCollectionInfo &signal ); /** * @brief Add new raw CAN Frame history buffer. If frame is not needed call will be just ignored @@ -195,61 +195,22 @@ class CollectionInspectionEngine std::array &buffer, uint8_t size ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA void setRawDataBufferManager( std::shared_ptr rawBufferManager ) { mRawBufferManager = std::move( rawBufferManager ); }; -#endif void setActiveDTCs( const DTCInfo &activeDTCs ); private: static const uint32_t MAX_SAMPLE_MEMORY = 20 * 1024 * 1024; // 20MB max for all samples - static inline InspectionValue + static inline double EVAL_EQUAL_DISTANCE() { return 0.001; } // because static const double (non-integral type) not possible - struct SampleConsumed - { - bool - isAlreadyConsumed( uint32_t conditionId ) - { - return mAlreadyConsumed.test( conditionId ); - } - void - setAlreadyConsumed( uint32_t conditionId, bool value ) - { - if ( conditionId == ALL_CONDITIONS ) - { - if ( value ) - { - mAlreadyConsumed.set(); - } - else - { - mAlreadyConsumed.reset(); - } - } - else - { - mAlreadyConsumed[conditionId] = value; - } - } - - private: - std::bitset mAlreadyConsumed{ 0 }; - }; - template - struct SignalSample : SampleConsumed - { - T mValue; - InspectionTimestamp mTimestamp{ 0 }; - }; - struct CanFrameSample : SampleConsumed { uint8_t mSize{ 0 }; /**< elements in buffer variable used. So if the raw can messages is only 3 bytes big this @@ -257,7 +218,7 @@ class CollectionInspectionEngine optimized sizeof(size)+sizeof(buffer) = 9 < sizeof(std:vector) = 24 so also for empty messages its mor efficient to preallocate 8 bytes even if not all will be used*/ std::array mBuffer{}; - InspectionTimestamp mTimestamp{ 0 }; + Timestamp mTimestamp{ 0 }; }; /** @@ -268,7 +229,7 @@ class CollectionInspectionEngine * need to look at historic values. The window is time based and not sample based. * Currently the last 2 windows are maintained inside this class. * */ - template + template class FixedTimeWindowFunctionData { public: @@ -277,8 +238,8 @@ class CollectionInspectionEngine { } - InspectionTimestamp mWindowSizeMs{ 0 }; /** ::min() }; @@ -309,12 +270,12 @@ class CollectionInspectionEngine * * @return true if any window value changed false otherwise */ - bool updateWindow( InspectionTimestamp timestamp, InspectionTimestamp &nextWindowFunctionTimesOut ); - inline void - addValue( T value, InspectionTimestamp timestamp, InspectionTimestamp &nextWindowFunctionTimesOut ) + bool updateWindow( Timestamp timestamp, Timestamp &nextWindowFunctionTimesOut ); + inline bool + addValue( T value, Timestamp timestamp, Timestamp &nextWindowFunctionTimesOut ) { - updateWindow( timestamp, nextWindowFunctionTimesOut ); updateInternalVariables( value ); + return updateWindow( timestamp, nextWindowFunctionTimesOut ); } private: @@ -327,7 +288,7 @@ class CollectionInspectionEngine mCollectedSignals++; } inline void - initNewWindow( InspectionTimestamp timestamp, InspectionTimestamp &nextWindowFunctionTimesOut ) + initNewWindow( Timestamp timestamp, Timestamp &nextWindowFunctionTimesOut ) { mCollectingMin = std::numeric_limits::max(); mCollectingMax = std::numeric_limits::min(); @@ -345,35 +306,25 @@ class CollectionInspectionEngine * The signal can be used as part of a condition or only to be published in the case * a condition is true */ - template + template struct SignalHistoryBuffer { - SignalHistoryBuffer( uint32_t sizeIn, - uint32_t sampleInterval -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - bool containsRawDataHandles = false -#endif - ) + SignalHistoryBuffer( uint32_t size, uint32_t sampleInterval, bool containsRawDataHandles = false ) : mMinimumSampleIntervalMs( sampleInterval ) - , mSize( sizeIn ) + , mSize( size ) , mCurrentPosition( mSize - 1 ) -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA , mContainsRawDataHandles( containsRawDataHandles ) -#endif { } uint32_t mMinimumSampleIntervalMs{ 0 }; std::vector> mBuffer; // ringbuffer, Consider to move to raw pointer allocated with new[] if vector allocates too much - uint32_t mSize{ 0 }; // minimum size needed by all conditions, buffer must be at least this big - uint32_t mCurrentPosition{ 0 }; /**< position in ringbuffer needs to come after size as it depends on it */ - uint32_t mCounter{ 0 }; /**< over all recorded samples*/ + size_t mSize{ 0 }; // minimum size needed by all conditions, buffer must be at least this big + size_t mCurrentPosition{ 0 }; /**< position in ringbuffer needs to come after size as it depends on it */ + size_t mCounter{ 0 }; /**< over all recorded samples*/ TimePoint mLastSample{ 0, 0 }; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA bool mContainsRawDataHandles{ false }; -#endif std::vector> mWindowFunctionData; /**< every signal buffer can have multiple windows over different time periods*/ std::bitset @@ -423,12 +374,12 @@ class CollectionInspectionEngine CanFrameHistoryBuffer() = default; CanFrameHistoryBuffer( CANRawFrameID frameID, CANChannelNumericID channelID, - uint32_t sizeIn, + uint32_t size, uint32_t sampleInterval ) : mFrameID( frameID ) , mChannelID( channelID ) , mMinimumSampleIntervalMs( sampleInterval ) - , mSize( sizeIn ) + , mSize( size ) , mCurrentPosition( mSize - 1 ) { } @@ -437,14 +388,14 @@ class CollectionInspectionEngine uint32_t mMinimumSampleIntervalMs{ 0 }; std::vector mBuffer; // ringbuffer, Consider to move to raw pointer allocated with new[] if vector allocates too much - uint32_t mSize{ 0 }; - uint32_t mCurrentPosition{ mSize - 1 }; // position in ringbuffer - uint32_t mCounter{ 0 }; + size_t mSize{ 0 }; + size_t mCurrentPosition{ mSize - 1 }; // position in ringbuffer + size_t mCounter{ 0 }; TimePoint mLastSample{ 0, 0 }; }; // VSS supported datatypes - using signalHistoryBufferPtrVar = boost::variant *, + using SignalHistoryBufferPtrVar = boost::variant *, SignalHistoryBuffer *, SignalHistoryBuffer *, SignalHistoryBuffer *, @@ -478,23 +429,24 @@ class CollectionInspectionEngine : mCondition( conditionIn ) { } - InspectionTimestamp mLastDataTimestampPublished{ 0 }; + Timestamp mLastDataTimestampPublished{ 0 }; TimePoint mLastTrigger{ 0, 0 }; - // TODO** :: Update the type here - std::unordered_map - mEvaluationSignals; // for fast lookup signals used for evaluation - std::unordered_map + std::unordered_map + mConditionSignals; // for fast lookup signals used for evaluation or collection + std::unordered_map mEvaluationFunctions; // for fast lookup functions used for evaluation const ConditionWithCollectedData &mCondition; // Unique Identifier of the Event matched by this condition. EventID mEventID{ 0 }; + std::unordered_set mCollectedSignalIds; + CollectionInspectionEngineOutput mCollectedData; - template + template SignalHistoryBuffer * - getEvaluationSignalsBufferPtr( InspectionSignalID signalIDIn ) + getEvaluationSignalsBufferPtr( SignalID signalID ) { - auto evaluationSignalPtr = mEvaluationSignals.find( signalIDIn ); - if ( evaluationSignalPtr == mEvaluationSignals.end() ) + auto evaluationSignalPtr = mConditionSignals.find( signalID ); + if ( evaluationSignalPtr == mConditionSignals.end() ) { return nullptr; } @@ -512,11 +464,11 @@ class CollectionInspectionEngine return nullptr; } - template + template FixedTimeWindowFunctionData * - getFixedTimeWindowFunctionDataPtr( InspectionSignalID signalIDIn ) + getFixedTimeWindowFunctionDataPtr( SignalID signalID ) { - auto evaluationfunctionPtr = mEvaluationFunctions.find( signalIDIn ); + auto evaluationfunctionPtr = mEvaluationFunctions.find( signalID ); if ( evaluationfunctionPtr == mEvaluationFunctions.end() ) { return nullptr; @@ -536,60 +488,52 @@ class CollectionInspectionEngine } }; - enum class ExpressionErrorCode - { - SUCCESSFUL, - SIGNAL_NOT_FOUND, - FUNCTION_DATA_NOT_AVAILABLE, - STACK_DEPTH_REACHED, - NOT_IMPLEMENTED_TYPE, - NOT_IMPLEMENTED_FUNCTION - }; - bool preAllocateBuffers(); - bool isSignalPartOfEval( const ExpressionNode *expression, InspectionSignalID signalID, int remainingStackDepth ); ExpressionErrorCode eval( const ExpressionNode *expression, ActiveCondition &condition, - InspectionValue &resultValueDouble, - bool &resultValueBool, - int remainingStackDepth ); - ExpressionErrorCode getLatestSignalValue( InspectionSignalID id, - ActiveCondition &condition, - InspectionValue &result ); + InspectionValue &resultValue, + int remainingStackDepth, + uint32_t conditionId ); + + /** + * @brief Evaluate static conditions once when new inspection matrix arrives + */ + void evaluateStaticCondition( uint32_t conditionIndex ); + + ExpressionErrorCode getLatestSignalValue( SignalID id, ActiveCondition &condition, InspectionValue &result ); ExpressionErrorCode getSampleWindowFunction( WindowFunction function, - InspectionSignalID signalID, + SignalID signalID, ActiveCondition &condition, InspectionValue &result ); template ExpressionErrorCode getSampleWindowFunctionType( WindowFunction function, - InspectionSignalID signalID, + SignalID signalID, ActiveCondition &condition, InspectionValue &result ); template - void collectLastSignals( InspectionSignalID id, - uint32_t minimumSamplingInterval, - uint32_t maxNumberOfSignalsToCollect, + void collectLastSignals( SignalID id, + size_t maxNumberOfSignalsToCollect, uint32_t conditionId, - SignalType signalTypeIn, - InspectionTimestamp &newestSignalTimestamp, + SignalType signalType, + Timestamp &newestSignalTimestamp, std::vector &output ); void collectLastCanFrames( CANRawFrameID canID, CANChannelNumericID channelID, uint32_t minimumSamplingInterval, - uint32_t maxNumberOfSignalsToCollect, + size_t maxNumberOfSignalsToCollect, uint32_t conditionId, - InspectionTimestamp &newestSignalTimestamp, + Timestamp &newestSignalTimestamp, std::vector &output ); template - void updateConditionBuffer( const InspectionMatrixSignalCollectionInfo &inspectionMatrixCollectionInfoIn, - ActiveCondition &acIn, - const long unsigned int conditionIndexIn ); + void updateConditionBuffer( const InspectionMatrixSignalCollectionInfo &inspectionMatrixCollectionInfo, + ActiveCondition &activeCondition, + const long unsigned int conditionIndex ); - void updateAllFixedWindowFunctions( InspectionTimestamp timestamp ); + void updateAllFixedWindowFunctions( Timestamp timestamp ); /** * @brief Generate a unique Identifier of an event. The event ID @@ -598,7 +542,7 @@ class CollectionInspectionEngine * @param timestamp in ms when the event occurred. * @return Unique Identifier of the event */ - static EventID generateEventID( InspectionTimestamp timestamp ); + static EventID generateEventID( Timestamp timestamp ); void clear(); @@ -614,72 +558,100 @@ class CollectionInspectionEngine } // VSS supported datatypes - using signalHistoryBufferVar = boost::variant>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>, - std::vector>>; - using SignalHistoryBufferCollection = std::unordered_map; + using SignalHistoryBuffersVar = boost::variant>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>, + std::vector>>; + using SignalHistoryBufferCollection = std::unordered_map; SignalHistoryBufferCollection mSignalBuffers; /**< signal history buffer. First vector has the signalID as index. In the nested vector * the different subsampling of this signal are stored. */ - using SignalToBufferTypeMap = std::unordered_map; + using SignalToBufferTypeMap = std::unordered_map; SignalToBufferTypeMap mSignalToBufferTypeMap; - template + /** + * @brief This function will either return existing signal buffer vector or create a new one + */ + template std::vector> * - getSignalHistoryBufferPtr( InspectionSignalID signalIDIn ) + getSignalHistoryBuffersPtr( SignalID signalID ) { std::vector> *resVec = nullptr; - if ( mSignalBuffers.find( signalIDIn ) == mSignalBuffers.end() ) + if ( mSignalBuffers.find( signalID ) == mSignalBuffers.end() ) { // create a new map entry auto mapEntryVec = std::vector>{}; try { - signalHistoryBufferVar mapEntry = mapEntryVec; - mSignalBuffers.insert( { signalIDIn, mapEntry } ); + SignalHistoryBuffersVar mapEntry = mapEntryVec; + FWE_LOG_TRACE( "Creating new signalHistoryBuffer vector for Signal " + std::to_string( signalID ) + + " with type " + boost::core::demangle( typeid( T ).name() ) ); + mSignalBuffers.insert( { signalID, mapEntry } ); } catch ( ... ) { FWE_LOG_ERROR( "Cannot Insert the signalHistoryBuffer vector for Signal " + - std::to_string( signalIDIn ) ); + std::to_string( signalID ) ); return nullptr; } } try { - auto signalBufferVectorPtr = mSignalBuffers.find( signalIDIn ); - if ( signalBufferVectorPtr != mSignalBuffers.end() ) + auto signalBufferVectorPtr = mSignalBuffers.find( signalID ); + resVec = boost::get>>( &( signalBufferVectorPtr->second ) ); + if ( resVec == nullptr ) { - resVec = boost::get>>( &( signalBufferVectorPtr->second ) ); + FWE_LOG_ERROR( "Could not retrieve the signalHistoryBuffer vector for Signal " + + std::to_string( signalID ) + + ". Tried with type: " + boost::core::demangle( typeid( T ).name() ) + + ". The buffer was likely created with the wrong type." ); } } catch ( ... ) { - FWE_LOG_ERROR( "Cannot retrive the signalHistoryBuffer vector for Signal " + std::to_string( signalIDIn ) ); + FWE_LOG_ERROR( "Cannot retrieve the signalHistoryBuffer vector for Signal " + std::to_string( signalID ) ); } return resVec; } + /** + * @brief This function will return existing signal buffer for given signal id and sample rate + */ template - bool allocateBufferVector( SignalID signalIDIn, uint32_t &usedBytes ); + SignalHistoryBuffer * + getSignalHistoryBufferPtr( SignalID signalID, uint32_t minimumSampleIntervalMs ) + { + auto signalHistoryBufferVectorPtr = getSignalHistoryBuffersPtr( signalID ); + if ( signalHistoryBufferVectorPtr != nullptr ) + { + for ( auto &buffer : *signalHistoryBufferVectorPtr ) + { + if ( buffer.mMinimumSampleIntervalMs == minimumSampleIntervalMs ) + { + return &buffer; + } + } + } + return nullptr; + } + + template + bool allocateBufferVector( SignalID signalID, size_t &usedBytes ); - template - void updateBufferFixedWindowFunction( SignalID signalIDIn, InspectionTimestamp timestamp ); + template + void updateBufferFixedWindowFunction( SignalID signalID, Timestamp timestamp ); template - ExpressionErrorCode getLatestBufferSignalValue( InspectionSignalID id, - ActiveCondition &condition, - InspectionValue &result ); + ExpressionErrorCode getLatestBufferSignalValue( SignalID id, ActiveCondition &condition, InspectionValue &result ); using CanFrameHistoryBufferCollection = std::vector; CanFrameHistoryBufferCollection mCanFrameBuffers; /**< signal history buffer for raw can frames. */ @@ -704,36 +676,38 @@ class CollectionInspectionEngine std::bitset mConditionsWithConditionCurrentlyTrue; // bit is set if the condition evaluated to true the last time std::bitset - mConditionsNotTriggeredWaitingPublished; // bit is set if condition is not triggered, if bit is not set it means - // condition is triggered and waits for its data to be sent out + mConditionsTriggeredWaitingPublished; // bit is set if condition is triggered and waits for its data to be sent + // out, bit is not set if condition is not triggered std::vector mConditions; std::shared_ptr mActiveInspectionMatrix; - std::shared_ptr collectData( ActiveCondition &condition, - uint32_t conditionId, - InspectionTimestamp &newestSignalTimestamp ); + void collectData( ActiveCondition &condition, + uint32_t conditionId, + Timestamp &newestSignalTimestamp, + CollectionInspectionEngineOutput &output ); uint32_t mNextConditionToCollectedIndex{ 0 }; - InspectionTimestamp mNextWindowFunctionTimesOut{ 0 }; + Timestamp mNextWindowFunctionTimesOut{ 0 }; bool mSendDataOnlyOncePerCondition{ false }; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA std::shared_ptr mRawBufferManager{ nullptr }; -#endif }; template void -CollectionInspectionEngine::addNewSignal( InspectionSignalID id, const TimePoint &receiveTime, T value ) +CollectionInspectionEngine::addNewSignal( SignalID id, + const TimePoint &receiveTime, + const Timestamp ¤tMonotonicTimeMs, + T value ) { - if ( mSignalBuffers.find( id ) == mSignalBuffers.end() || mSignalBuffers[id].empty() ) + if ( mSignalBuffers.find( id ) == mSignalBuffers.end() ) { // Signal not collected by any active condition return; } // Iterate through all sampling intervals of the signal std::vector> *signalHistoryBufferPtr = nullptr; - signalHistoryBufferPtr = getSignalHistoryBufferPtr( id ); + signalHistoryBufferPtr = getSignalHistoryBuffersPtr( id ); if ( signalHistoryBufferPtr == nullptr ) { // Invalid access to the map Buffer datatype @@ -742,17 +716,19 @@ CollectionInspectionEngine::addNewSignal( InspectionSignalID id, const TimePoint auto &bufferVec = *signalHistoryBufferPtr; for ( auto &buf : bufferVec ) { - if ( ( ( buf.mSize > 0 ) && ( buf.mSize <= buf.mBuffer.size() ) ) && - ( ( buf.mMinimumSampleIntervalMs == 0 ) || - ( ( buf.mLastSample.systemTimeMs == 0 ) && ( buf.mLastSample.monotonicTimeMs == 0 ) ) || - ( receiveTime.monotonicTimeMs >= buf.mLastSample.monotonicTimeMs + buf.mMinimumSampleIntervalMs ) ) ) + if ( ( ( buf.mSize > 0 ) && ( buf.mSize <= buf.mBuffer.size() ) ) && // buffer isn't full + ( ( buf.mMinimumSampleIntervalMs == 0 ) || // sample rate is 0 = all incoming signals are collected + ( ( buf.mLastSample.systemTimeMs == 0 ) && + ( buf.mLastSample.monotonicTimeMs == 0 ) ) || // first sample that is collected + ( receiveTime.monotonicTimeMs >= + buf.mLastSample.monotonicTimeMs + buf.mMinimumSampleIntervalMs ) ) ) // sample time has passed { + auto oldValue = buf.mBuffer[buf.mCurrentPosition].mValue; buf.mCurrentPosition++; if ( buf.mCurrentPosition >= buf.mSize ) { buf.mCurrentPosition = 0; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA if ( buf.mContainsRawDataHandles && ( buf.mCounter >= buf.mSize ) ) { // release data that is going to be overwritten @@ -762,8 +738,7 @@ CollectionInspectionEngine::addNewSignal( InspectionSignalID id, const TimePoint RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_HISTORY_BUFFER, buf.mBuffer[buf.mCurrentPosition].mValue ); } -#endif - + // insert value to the buffer buf.mBuffer[buf.mCurrentPosition].mValue = value; buf.mBuffer[buf.mCurrentPosition].mTimestamp = receiveTime.systemTimeMs; buf.mBuffer[buf.mCurrentPosition].setAlreadyConsumed( ALL_CONDITIONS, false ); @@ -771,10 +746,16 @@ CollectionInspectionEngine::addNewSignal( InspectionSignalID id, const TimePoint buf.mLastSample = receiveTime; for ( auto &window : buf.mWindowFunctionData ) { - window.addValue( value, receiveTime.monotonicTimeMs, mNextWindowFunctionTimesOut ); + if ( window.addValue( value, currentMonotonicTimeMs, mNextWindowFunctionTimesOut ) ) + { + // Window function values were recalculated + mConditionsWithInputSignalChanged |= buf.mConditionsThatEvaluateOnThisSignal; + } + } + if ( oldValue != value ) + { + mConditionsWithInputSignalChanged |= buf.mConditionsThatEvaluateOnThisSignal; } - mConditionsWithInputSignalChanged |= buf.mConditionsThatEvaluateOnThisSignal; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA if ( buf.mContainsRawDataHandles ) { NotifyRawBufferManager::increaseElementUsage( @@ -783,15 +764,14 @@ CollectionInspectionEngine::addNewSignal( InspectionSignalID id, const TimePoint RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_HISTORY_BUFFER, value ); } -#endif } } } template bool -CollectionInspectionEngine::FixedTimeWindowFunctionData::updateWindow( - InspectionTimestamp timestamp, InspectionTimestamp &nextWindowFunctionTimesOut ) +CollectionInspectionEngine::FixedTimeWindowFunctionData::updateWindow( Timestamp timestamp, + Timestamp &nextWindowFunctionTimesOut ) { if ( mLastTimeCalculated == 0 ) { diff --git a/src/CollectionInspectionWorkerThread.cpp b/src/CollectionInspectionWorkerThread.cpp index e2dbd00e..3318217f 100644 --- a/src/CollectionInspectionWorkerThread.cpp +++ b/src/CollectionInspectionWorkerThread.cpp @@ -4,6 +4,7 @@ #include "CollectionInspectionWorkerThread.h" #include "CANDataTypes.h" #include "LoggingModule.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "TraceModule.h" #include @@ -19,101 +20,92 @@ namespace IoTFleetWise bool CollectionInspectionWorkerThread::init( const std::shared_ptr &inputSignalBuffer, - const std::shared_ptr &outputCollectedData, - uint32_t idleTimeMs -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::shared_ptr rawBufferManager -#endif -) + const std::shared_ptr &outputCollectedData, + uint32_t idleTimeMs, + std::shared_ptr rawBufferManager ) { - fInputSignalBuffer = inputSignalBuffer; - fOutputCollectedData = outputCollectedData; + mInputSignalBuffer = inputSignalBuffer; + mOutputCollectedData = outputCollectedData; if ( idleTimeMs != 0 ) { - fIdleTimeMs = idleTimeMs; + mIdleTimeMs = idleTimeMs; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - fCollectionInspectionEngine.setRawDataBufferManager( rawBufferManager ); + mCollectionInspectionEngine.setRawDataBufferManager( rawBufferManager ); mRawBufferManager = std::move( rawBufferManager ); -#endif return true; } bool CollectionInspectionWorkerThread::start() { - if ( ( fInputSignalBuffer == nullptr ) || ( fOutputCollectedData == nullptr ) ) + if ( ( mInputSignalBuffer == nullptr ) || ( mOutputCollectedData == nullptr ) ) { FWE_LOG_ERROR( "Collection Engine cannot be started without correct configurations" ); return false; } // Prevent concurrent stop/init - std::lock_guard lock( fThreadMutex ); - // On multi core systems the shared variable fShouldStop must be updated for + std::lock_guard lock( mThreadMutex ); + // On multi core systems the shared variable mShouldStop must be updated for // all cores before starting the thread otherwise thread will directly end - fShouldStop.store( false ); - if ( !fThread.create( doWork, this ) ) + mShouldStop.store( false ); + if ( !mThread.create( doWork, this ) ) { FWE_LOG_TRACE( "Inspection Thread failed to start" ); } else { FWE_LOG_TRACE( "Inspection Thread started" ); - fThread.setThreadName( "fwDICollInsEng" ); + mThread.setThreadName( "fwDICollInsEng" ); } - return fThread.isActive() && fThread.isValid(); + return mThread.isActive() && mThread.isValid(); } bool CollectionInspectionWorkerThread::stop() { - if ( ( !fThread.isValid() ) || ( !fThread.isActive() ) ) + if ( ( !mThread.isValid() ) || ( !mThread.isActive() ) ) { return true; } - std::lock_guard lock( fThreadMutex ); - fShouldStop.store( true, std::memory_order_relaxed ); + std::lock_guard lock( mThreadMutex ); + mShouldStop.store( true, std::memory_order_relaxed ); FWE_LOG_TRACE( "Request stop" ); - fWait.notify(); - fThread.release(); + mWait.notify(); + mThread.release(); FWE_LOG_TRACE( "Stop finished" ); - fShouldStop.store( false, std::memory_order_relaxed ); - return !fThread.isActive(); + mShouldStop.store( false, std::memory_order_relaxed ); + return !mThread.isActive(); } bool CollectionInspectionWorkerThread::shouldStop() const { - return fShouldStop.load( std::memory_order_relaxed ); + return mShouldStop.load( std::memory_order_relaxed ); } void CollectionInspectionWorkerThread::onChangeInspectionMatrix( const std::shared_ptr &inspectionMatrix ) { - { - std::lock_guard lock( fInspectionMatrixMutex ); - fUpdatedInspectionMatrix = inspectionMatrix; - fUpdatedInspectionMatrixAvailable = true; - FWE_LOG_TRACE( "New inspection matrix handed over" ); - // Wake up the thread. - fWait.notify(); - } + std::lock_guard lock( mInspectionMatrixMutex ); + mUpdatedInspectionMatrix = inspectionMatrix; + mUpdatedInspectionMatrixAvailable = true; + FWE_LOG_TRACE( "New inspection matrix handed over" ); + // Wake up the thread. + mWait.notify(); } void CollectionInspectionWorkerThread::onNewDataAvailable() { - fWait.notify(); + mWait.notify(); } void CollectionInspectionWorkerThread::doWork( void *data ) { - CollectionInspectionWorkerThread *consumer = static_cast( data ); TimePoint lastTimeEvaluated = { 0, 0 }; Timestamp lastTraceOutput = 0; @@ -123,29 +115,28 @@ CollectionInspectionWorkerThread::doWork( void *data ) while ( true ) { activations++; - if ( consumer->fUpdatedInspectionMatrixAvailable ) + if ( consumer->mUpdatedInspectionMatrixAvailable ) { std::shared_ptr newInspectionMatrix; { - std::lock_guard lock( consumer->fInspectionMatrixMutex ); - consumer->fUpdatedInspectionMatrixAvailable = false; - newInspectionMatrix = consumer->fUpdatedInspectionMatrix; + std::lock_guard lock( consumer->mInspectionMatrixMutex ); + consumer->mUpdatedInspectionMatrixAvailable = false; + newInspectionMatrix = consumer->mUpdatedInspectionMatrix; } - consumer->fCollectionInspectionEngine.onChangeInspectionMatrix( newInspectionMatrix, - consumer->fClock->timeSinceEpoch() ); + + consumer->mCollectionInspectionEngine.onChangeInspectionMatrix( newInspectionMatrix, + consumer->mClock->timeSinceEpoch() ); } // Only run the main inspection loop if there is an inspection matrix // Otherwise, go to sleep. - if ( consumer->fUpdatedInspectionMatrix ) + if ( consumer->mUpdatedInspectionMatrix ) { std::array buf = {}; CollectedCanRawFrame inputCANFrame( 0, 0, 0, buf, 0 ); - TimePoint currentTime = consumer->fClock->timeSinceEpoch(); - uint32_t waitTimeMs = consumer->fIdleTimeMs; + TimePoint currentTime = consumer->mClock->timeSinceEpoch(); + uint32_t waitTimeMs = consumer->mIdleTimeMs; // Consume any new signals and pass them over to the inspection Engine auto consumeSignalGroups = [&]( const CollectedDataFrame &dataFrame ) { - TraceModule::get().decrementAtomicVariable( - TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); TraceModule::get().incrementVariable( TraceVariable::CE_PROCESSED_DATA_FRAMES ); if ( !dataFrame.mCollectedSignals.empty() ) @@ -159,76 +150,92 @@ CollectionInspectionWorkerThread::doWork( void *data ) switch ( signalValue.getType() ) { case SignalType::UINT8: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.uint8Val ); break; case SignalType::INT8: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.int8Val ); break; case SignalType::UINT16: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.uint16Val ); break; case SignalType::INT16: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.int16Val ); break; case SignalType::UINT32: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.uint32Val ); break; case SignalType::INT32: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.int32Val ); break; case SignalType::UINT64: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.uint64Val ); break; case SignalType::INT64: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.int64Val ); break; case SignalType::FLOAT: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.floatVal ); break; case SignalType::DOUBLE: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.doubleVal ); break; case SignalType::BOOLEAN: - consumer->fCollectionInspectionEngine.addNewSignal( + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.boolVal ); break; + case SignalType::UNKNOWN: + FWE_LOG_WARN( "UNKNOWN signal [signal id: " + std::to_string( inputSignal.signalID ) + + " ] should not be processed" ); + break; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: - consumer->fCollectionInspectionEngine.addNewSignal( + case SignalType::COMPLEX_SIGNAL: + consumer->mCollectionInspectionEngine.addNewSignal( inputSignal.signalID, calculateMonotonicTime( currentTime, inputSignal.receiveTime ), + currentTime.monotonicTimeMs, signalValue.value.uint32Val ); if ( consumer->mRawBufferManager != nullptr ) { @@ -249,7 +256,7 @@ CollectionInspectionWorkerThread::doWork( void *data ) TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_CAN ); TraceModule::get().incrementVariable( TraceVariable::CE_PROCESSED_CAN_FRAMES ); - consumer->fCollectionInspectionEngine.addNewRawCanFrame( + consumer->mCollectionInspectionEngine.addNewRawCanFrame( dataFrame.mCollectedCanRawFrame->frameID, dataFrame.mCollectedCanRawFrame->channelId, calculateMonotonicTime( currentTime, dataFrame.mCollectedCanRawFrame->receiveTime ), @@ -270,32 +277,32 @@ CollectionInspectionWorkerThread::doWork( void *data ) TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DTCS ); TraceModule::get().incrementVariable( TraceVariable::CE_PROCESSED_DTCS ); - consumer->fCollectionInspectionEngine.setActiveDTCs( *dataFrame.mActiveDTCs.get() ); + consumer->mCollectionInspectionEngine.setActiveDTCs( *dataFrame.mActiveDTCs.get() ); statisticInputMessagesProcessed++; } - lastTimeEvaluated = consumer->fClock->timeSinceEpoch(); - consumer->fCollectionInspectionEngine.evaluateConditions( lastTimeEvaluated ); + lastTimeEvaluated = consumer->mClock->timeSinceEpoch(); + consumer->mCollectionInspectionEngine.evaluateConditions( lastTimeEvaluated ); // Initiate data collection and upload after every condition evaluation statisticDataSentOut += consumer->collectDataAndUpload(); }; - auto consumed = consumer->fInputSignalBuffer->consumeAll( consumeSignalGroups ); + auto consumed = consumer->mInputSignalBuffer->consumeAll( consumeSignalGroups ); // If nothing was consumed and at least the evaluate interval has elapsed, evaluate the // conditions to check heartbeat campaigns: - if ( ( consumed == 0 ) && ( ( consumer->fClock->monotonicTimeSinceEpochMs() - + if ( ( consumed == 0 ) && ( ( consumer->mClock->monotonicTimeSinceEpochMs() - lastTimeEvaluated.monotonicTimeMs ) >= EVALUATE_INTERVAL_MS ) ) { - lastTimeEvaluated = consumer->fClock->timeSinceEpoch(); - consumer->fCollectionInspectionEngine.evaluateConditions( lastTimeEvaluated ); + lastTimeEvaluated = consumer->mClock->timeSinceEpoch(); + consumer->mCollectionInspectionEngine.evaluateConditions( lastTimeEvaluated ); statisticDataSentOut += consumer->collectDataAndUpload(); } // Nothing is in the ring buffer to consume. Go to idle mode for some time. - uint32_t timeToWait = std::min( waitTimeMs, consumer->fIdleTimeMs ); + uint32_t timeToWait = std::min( waitTimeMs, consumer->mIdleTimeMs ); // Print only every THREAD_IDLE_TIME_MS to avoid console spam - if ( consumer->fClock->monotonicTimeSinceEpochMs() > + if ( consumer->mClock->monotonicTimeSinceEpochMs() > ( lastTraceOutput + LoggingModule::LOG_AGGREGATION_TIME_MS ) ) { FWE_LOG_TRACE( "Activations: " + std::to_string( activations ) + @@ -307,14 +314,14 @@ CollectionInspectionWorkerThread::doWork( void *data ) activations = 0; statisticInputMessagesProcessed = 0; statisticDataSentOut = 0; - lastTraceOutput = consumer->fClock->monotonicTimeSinceEpochMs(); + lastTraceOutput = consumer->mClock->monotonicTimeSinceEpochMs(); } - consumer->fWait.wait( timeToWait ); + consumer->mWait.wait( timeToWait ); } else { // No inspection Matrix available. Wait for it from the CollectionScheme manager - consumer->fWait.wait( Signal::WaitWithPredicate ); + consumer->mWait.wait( Signal::WaitWithPredicate ); } if ( consumer->shouldStop() ) { @@ -327,23 +334,47 @@ uint32_t CollectionInspectionWorkerThread::collectDataAndUpload() { uint32_t collectedDataPackages = 0; - uint32_t waitTimeMs = this->fIdleTimeMs; - std::shared_ptr collectedData = - this->fCollectionInspectionEngine.collectNextDataToSend( this->fClock->timeSinceEpoch(), waitTimeMs ); - while ( ( collectedData != nullptr ) && ( !this->shouldStop() ) ) + uint32_t waitTimeMs = this->mIdleTimeMs; + auto collectedData = + this->mCollectionInspectionEngine.collectNextDataToSend( this->mClock->timeSinceEpoch(), waitTimeMs ); + while ( ( ( collectedData.triggeredCollectionSchemeData != nullptr ) +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + || ( collectedData.triggeredVisionSystemData != nullptr ) +#endif + ) && + ( !this->shouldStop() ) ) { TraceModule::get().incrementVariable( TraceVariable::CE_TRIGGERS ); - if ( !this->fOutputCollectedData->push( std::move( collectedData ) ) ) + if ( collectedData.triggeredCollectionSchemeData != nullptr ) { - FWE_LOG_WARN( "Collected data output buffer is full" ); + { + if ( this->mOutputCollectedData->push( collectedData.triggeredCollectionSchemeData ) ) + { + collectedDataPackages++; + } + else + { + FWE_LOG_WARN( "Collected data output buffer is full" ); + } + } } - else + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + if ( collectedData.triggeredVisionSystemData != nullptr ) { - collectedDataPackages++; - this->mDataReadyListeners.notify(); + if ( this->mOutputCollectedData->push( collectedData.triggeredVisionSystemData ) ) + { + collectedDataPackages++; + } + else + { + FWE_LOG_WARN( "Collected data output buffer is full, Vision System Data could not be pushed" ); + } } +#endif + collectedData = - this->fCollectionInspectionEngine.collectNextDataToSend( this->fClock->timeSinceEpoch(), waitTimeMs ); + this->mCollectionInspectionEngine.collectNextDataToSend( this->mClock->timeSinceEpoch(), waitTimeMs ); } return collectedDataPackages; } @@ -366,7 +397,7 @@ CollectionInspectionWorkerThread::calculateMonotonicTime( const TimePoint &currT bool CollectionInspectionWorkerThread::isAlive() { - return fThread.isValid() && fThread.isActive(); + return mThread.isValid() && mThread.isActive(); } CollectionInspectionWorkerThread::~CollectionInspectionWorkerThread() diff --git a/src/CollectionInspectionWorkerThread.h b/src/CollectionInspectionWorkerThread.h index 82101fd3..f8cb0662 100644 --- a/src/CollectionInspectionWorkerThread.h +++ b/src/CollectionInspectionWorkerThread.h @@ -7,20 +7,16 @@ #include "ClockHandler.h" #include "CollectionInspectionAPITypes.h" #include "CollectionInspectionEngine.h" -#include "Listener.h" +#include "DataSenderTypes.h" +#include "RawDataManager.h" #include "Signal.h" #include "Thread.h" #include "TimeTypes.h" #include #include -#include #include #include -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "RawDataManager.h" -#endif - namespace Aws { namespace IoTFleetWise @@ -29,9 +25,8 @@ namespace IoTFleetWise class CollectionInspectionWorkerThread { public: - using OnDataReadyToPublishCallback = std::function; - - CollectionInspectionWorkerThread() = default; + CollectionInspectionWorkerThread( CollectionInspectionEngine &collectionInspectionEngine ) + : mCollectionInspectionEngine( collectionInspectionEngine ){}; ~CollectionInspectionWorkerThread(); CollectionInspectionWorkerThread( const CollectionInspectionWorkerThread & ) = delete; @@ -41,15 +36,6 @@ class CollectionInspectionWorkerThread void onChangeInspectionMatrix( const std::shared_ptr &inspectionMatrix ); - /** - * @brief Register a callback to be called when data is ready to be published to the cloud - * */ - void - subscribeToDataReadyToPublish( OnDataReadyToPublishCallback callback ) - { - mDataReadyListeners.subscribe( callback ); - } - /** * @brief As soon as new data is available in any input queue call this to wakeup the thread * */ @@ -62,14 +48,11 @@ class CollectionInspectionWorkerThread * */ bool init( const std::shared_ptr &inputSignalBuffer, /**< IVehicleDataSourceConsumer instances will put relevant signals in this queue */ - const std::shared_ptr + const std::shared_ptr &outputCollectedData, /**< this thread will put data that should be sent to cloud into this queue */ - uint32_t idleTimeMs /**< if no new data is available sleep for this amount of milliseconds */ -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , + uint32_t idleTimeMs, /**< if no new data is available sleep for this amount of milliseconds */ std::shared_ptr rawBufferManager = nullptr /**< the raw buffer manager which is informed what data is used */ -#endif ); /** @@ -109,23 +92,20 @@ class CollectionInspectionWorkerThread */ uint32_t collectDataAndUpload(); - CollectionInspectionEngine fCollectionInspectionEngine; - - std::shared_ptr fInputSignalBuffer; - std::shared_ptr fOutputCollectedData; - Thread fThread; - std::atomic fShouldStop{ false }; - std::atomic fUpdatedInspectionMatrixAvailable{ false }; - std::shared_ptr fUpdatedInspectionMatrix; - std::mutex fInspectionMatrixMutex; - std::mutex fThreadMutex; - Signal fWait; - uint32_t fIdleTimeMs{ DEFAULT_THREAD_IDLE_TIME_MS }; - std::shared_ptr fClock = ClockHandler::getClock(); - ThreadSafeListeners mDataReadyListeners; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + CollectionInspectionEngine &mCollectionInspectionEngine; + + std::shared_ptr mInputSignalBuffer; + std::shared_ptr mOutputCollectedData; + Thread mThread; + std::atomic mShouldStop{ false }; + std::atomic mUpdatedInspectionMatrixAvailable{ false }; + std::shared_ptr mUpdatedInspectionMatrix; + std::mutex mInspectionMatrixMutex; + std::mutex mThreadMutex; + Signal mWait; + uint32_t mIdleTimeMs{ DEFAULT_THREAD_IDLE_TIME_MS }; + std::shared_ptr mClock = ClockHandler::getClock(); std::shared_ptr mRawBufferManager{ nullptr }; -#endif }; } // namespace IoTFleetWise diff --git a/src/CollectionSchemeIngestion.cpp b/src/CollectionSchemeIngestion.cpp index 4ba6a02c..bc607c08 100644 --- a/src/CollectionSchemeIngestion.cpp +++ b/src/CollectionSchemeIngestion.cpp @@ -5,12 +5,13 @@ #include "CollectionInspectionAPITypes.h" #include "LoggingModule.h" #include +#include +#include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA #include "MessageTypes.h" #include #include -#include #endif namespace Aws @@ -28,6 +29,26 @@ CollectionSchemeIngestion::~CollectionSchemeIngestion() google::protobuf::ShutdownProtobufLibrary(); } +bool +CollectionSchemeIngestion::operator==( const ICollectionScheme &other ) const +{ + bool isEqual = isReady() && other.isReady() && ( getCollectionSchemeID() == other.getCollectionSchemeID() ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // For Vision System Data, comparing just the ID isn't enough, because even though schemes with the same ID + // should be identical from the perspective of the Cloud, FWE generates some additional data internally which can + // change when it ingests a new message. + isEqual = + isEqual && ( getPartialSignalIdToSignalPathLookupTable() == other.getPartialSignalIdToSignalPathLookupTable() ); +#endif + return isEqual; +} + +bool +CollectionSchemeIngestion::operator!=( const ICollectionScheme &other ) const +{ + return !( *this == other ); +} + bool CollectionSchemeIngestion::isReady() const { @@ -129,12 +150,13 @@ CollectionSchemeIngestion::build() FWE_LOG_INFO( "CollectionScheme is Condition Based. Building AST with " + std::to_string( numNodes ) + " nodes" ); - mExpressionNodes.resize( numNodes ); + mExpressionNodes.reserve( numNodes ); // As pointers to elements inside the vector are used after this no realloc for mExpressionNodes is allowed std::size_t currentIndex = 0; // start at index 0 of mExpressionNodes for first node mExpressionNode = serializeNode( mProtoCollectionSchemeMessagePtr->condition_based_collection_scheme().condition_tree(), + mExpressionNodes, currentIndex, MAX_EQUATION_DEPTH ); FWE_LOG_INFO( "AST complete" ); @@ -192,7 +214,7 @@ CollectionSchemeIngestion::getOrInsertPartialSignalId( SignalID signalId, { // Search if path already exists - for ( auto &partialSignal : mPartialSignalIDLookup ) + for ( auto &partialSignal : *mPartialSignalIDLookup ) { if ( partialSignal.second.first == signalId ) { @@ -226,7 +248,7 @@ CollectionSchemeIngestion::getOrInsertPartialSignalId( SignalID signalId, // coverity[autosar_cpp14_m5_2_10_violation] For std::atomic this must be performed in a single statement PartialSignalID newPartialSignalId = mPartialSignalCounter++ | INTERNAL_SIGNAL_ID_BITMASK; - mPartialSignalIDLookup[newPartialSignalId] = std::pair( signalId, signal_path ); + ( *mPartialSignalIDLookup )[newPartialSignalId] = std::pair( signalId, signal_path ); return newPartialSignalId; } @@ -238,7 +260,7 @@ CollectionSchemeIngestion::getPartialSignalIdToSignalPathLookupTable() const { return INVALID_PARTIAL_SIGNAL_ID_LOOKUP; } - return mPartialSignalIDLookup; + return *mPartialSignalIDLookup; } #endif @@ -247,7 +269,7 @@ CollectionSchemeIngestion::getAllExpressionNodes() const { if ( !mReady ) { - return INVALID_EXPRESSION_NODE; + return INVALID_EXPRESSION_NODES; } return mExpressionNodes; @@ -278,7 +300,7 @@ CollectionSchemeIngestion::convertFunctionType( FWE_LOG_INFO( "Converting node to: PREV_LAST_FIXED_WINDOW_AVG" ); return WindowFunction::PREV_LAST_FIXED_WINDOW_AVG; default: - FWE_LOG_ERROR( "Function node type not supported." ); + FWE_LOG_ERROR( "WindowFunction node type not supported." ); return WindowFunction::NONE; } } @@ -357,6 +379,7 @@ CollectionSchemeIngestion::getNumberOfNodes( const Schemas::CommonTypesMsg::Cond ExpressionNode * CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::ConditionNode &node, + ExpressionNode_t &expressionNodes, std::size_t &nextIndex, int remainingDepth ) { @@ -364,15 +387,15 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi { return nullptr; } - mExpressionNodes.emplace_back(); + expressionNodes.emplace_back(); ExpressionNode *currentNode = nullptr; - if ( mExpressionNodes.empty() || ( mExpressionNodes.size() <= nextIndex ) ) + if ( expressionNodes.empty() || ( expressionNodes.size() <= nextIndex ) ) { return nullptr; } else { - currentNode = &( mExpressionNodes[nextIndex] ); + currentNode = &( expressionNodes[nextIndex] ); } nextIndex++; @@ -430,7 +453,7 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi } currentNode->function.windowFunction = convertFunctionType( node.node_function().window_function().window_type() ); - currentNode->nodeType = ExpressionNodeType::WINDOWFUNCTION; + currentNode->nodeType = ExpressionNodeType::WINDOW_FUNCTION; FWE_LOG_TRACE( "Creating Window FUNCTION node for Signal ID:" + std::to_string( currentNode->signalID ) ); return currentNode; } @@ -446,14 +469,8 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi { currentNode->nodeType = convertOperatorType( node.node_operator().operator_() ); FWE_LOG_TRACE( "Processing left child" ); - // If no valid function found return always false - if ( currentNode->nodeType == ExpressionNodeType::BOOLEAN ) - { - FWE_LOG_INFO( "Setting BOOLEAN node to false" ); - currentNode->booleanValue = false; - return currentNode; - } - ExpressionNode *left = serializeNode( node.node_operator().left_child(), nextIndex, remainingDepth - 1 ); + ExpressionNode *left = + serializeNode( node.node_operator().left_child(), expressionNodes, nextIndex, remainingDepth - 1 ); currentNode->left = left; // Not operator is unary and has only left child if ( ( currentNode->nodeType != ExpressionNodeType::OPERATOR_LOGICAL_NOT ) && @@ -461,7 +478,7 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi { FWE_LOG_TRACE( "Processing right child" ); ExpressionNode *right = - serializeNode( node.node_operator().right_child(), nextIndex, remainingDepth - 1 ); + serializeNode( node.node_operator().right_child(), expressionNodes, nextIndex, remainingDepth - 1 ); currentNode->right = right; } else @@ -469,8 +486,18 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi FWE_LOG_TRACE( "Setting right child to nullptr" ); currentNode->right = nullptr; } - return currentNode; + + if ( ( currentNode->nodeType != ExpressionNodeType::BOOLEAN ) && ( currentNode->left != nullptr ) && + ( ( currentNode->nodeType == ExpressionNodeType::OPERATOR_LOGICAL_NOT ) || + ( currentNode->right != nullptr ) ) ) + { + FWE_LOG_TRACE( "Creating Operator node" ); + + return currentNode; + } } + + FWE_LOG_WARN( "Invalid Operator node" ); } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA else if ( node.node_case() == Schemas::CommonTypesMsg::ConditionNode::kNodePrimitiveTypeInSignal ) @@ -497,11 +524,11 @@ CollectionSchemeIngestion::serializeNode( const Schemas::CommonTypesMsg::Conditi #endif // if not returned until here its an invalid node so remove it from buffer - mExpressionNodes.pop_back(); + expressionNodes.pop_back(); return nullptr; } -const std::string & +const SyncID & CollectionSchemeIngestion::getCollectionSchemeID() const { if ( !mReady ) @@ -512,7 +539,7 @@ CollectionSchemeIngestion::getCollectionSchemeID() const return mProtoCollectionSchemeMessagePtr->campaign_sync_id(); } -const std::string & +const SyncID & CollectionSchemeIngestion::getDecoderManifestID() const { if ( !mReady ) diff --git a/src/CollectionSchemeIngestion.h b/src/CollectionSchemeIngestion.h index 3da06c96..b20ae2c1 100644 --- a/src/CollectionSchemeIngestion.h +++ b/src/CollectionSchemeIngestion.h @@ -4,16 +4,16 @@ #pragma once #include "ICollectionScheme.h" +#include "SignalTypes.h" #include "collection_schemes.pb.h" #include "common_types.pb.h" #include #include #include -#include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "SignalTypes.h" #include +#include #endif namespace Aws @@ -28,7 +28,14 @@ namespace IoTFleetWise class CollectionSchemeIngestion : public ICollectionScheme { public: +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + CollectionSchemeIngestion( std::shared_ptr partialSignalIDLookup ) + : mPartialSignalIDLookup( std::move( partialSignalIDLookup ) ) + { + } +#else CollectionSchemeIngestion() = default; +#endif ~CollectionSchemeIngestion() override; CollectionSchemeIngestion( const CollectionSchemeIngestion & ) = delete; @@ -36,15 +43,19 @@ class CollectionSchemeIngestion : public ICollectionScheme CollectionSchemeIngestion( CollectionSchemeIngestion && ) = delete; CollectionSchemeIngestion &operator=( CollectionSchemeIngestion && ) = delete; + bool operator==( const ICollectionScheme &other ) const override; + + bool operator!=( const ICollectionScheme &other ) const override; + bool isReady() const override; bool build() override; bool copyData( std::shared_ptr protoCollectionSchemeMessagePtr ); - const std::string &getCollectionSchemeID() const override; + const SyncID &getCollectionSchemeID() const override; - const std::string &getDecoderManifestID() const override; + const SyncID &getDecoderManifestID() const override; uint64_t getStartTime() const override; @@ -120,7 +131,7 @@ class CollectionSchemeIngestion : public ICollectionScheme /** * @brief unordered_map from partial signal ID to pair of signal path and signal ID */ - PartialSignalIDLookup mPartialSignalIDLookup; + std::shared_ptr mPartialSignalIDLookup; /** * @brief Required metadata for S3 upload @@ -132,6 +143,7 @@ class CollectionSchemeIngestion : public ICollectionScheme * @brief Function used to Flatten the Abstract Syntax Tree (AST) */ ExpressionNode *serializeNode( const Schemas::CommonTypesMsg::ConditionNode &node, + ExpressionNode_t &expressionNodes, std::size_t &nextIndex, int remainingDepth ); diff --git a/src/CollectionSchemeIngestionList.cpp b/src/CollectionSchemeIngestionList.cpp index 6e3e1291..4fcf6f60 100644 --- a/src/CollectionSchemeIngestionList.cpp +++ b/src/CollectionSchemeIngestionList.cpp @@ -111,13 +111,18 @@ CollectionSchemeIngestionList::build() #ifdef FWE_FEATURE_VISION_SYSTEM_DATA // Reset partial signal id counter CollectionSchemeIngestion::mPartialSignalCounter = 0; + auto partialSignalIDLookup = std::make_shared(); #endif // Iterate through all the collectionSchemes in the collectionScheme list, make a shared pointer of them for ( int i = 0; i < mCollectionSchemeListMsg.collection_schemes_size(); i++ ) { // Create a CollectionSchemeIngestion Pointer, or pICPPtr. - auto pICPPtr = std::make_shared(); + auto pICPPtr = std::make_shared( +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + partialSignalIDLookup +#endif + ); // Stuff the pointer with the collectionScheme proto message data pICPPtr->copyData( std::make_shared( diff --git a/src/CollectionSchemeManager.cpp b/src/CollectionSchemeManager.cpp index 002cab4e..30919f82 100644 --- a/src/CollectionSchemeManager.cpp +++ b/src/CollectionSchemeManager.cpp @@ -5,7 +5,7 @@ #include "ICollectionScheme.h" #include "LoggingModule.h" #include "TraceModule.h" -#include +#include #include #include #include @@ -16,18 +16,14 @@ namespace Aws namespace IoTFleetWise { -const std::string CollectionSchemeManager::CHECKIN = "Checkin"; -CollectionSchemeManager::CollectionSchemeManager( std::string dm_id ) - : mCurrentDecoderManifestID( std::move( dm_id ) ) -{ -} - -CollectionSchemeManager::CollectionSchemeManager( std::string dm_id, - std::map mapEnabled, - std::map mapIdle ) - : mIdleCollectionSchemeMap( std::move( mapIdle ) ) - , mEnabledCollectionSchemeMap( std::move( mapEnabled ) ) - , mCurrentDecoderManifestID( std::move( dm_id ) ) +CollectionSchemeManager::CollectionSchemeManager( std::shared_ptr schemaPersistencyPtr, + CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr checkinSender, + std::shared_ptr rawDataBufferManager ) + : mCheckinSender( std::move( checkinSender ) ) + , mRawDataBufferManager( std::move( rawDataBufferManager ) ) + , mSchemaPersistency( std::move( schemaPersistencyPtr ) ) + , mCANIDTranslator( canIDTranslator ) { } @@ -90,7 +86,7 @@ CollectionSchemeManager::shouldStop() const /* supporting functions for logging */ void CollectionSchemeManager::printEventLogMsg( std::string &msg, - const std::string &id, + const SyncID &id, const Timestamp &startTime, const Timestamp &stopTime, const TimePoint &currTime ) @@ -124,59 +120,30 @@ CollectionSchemeManager::printWakeupStatus( std::string &wakeupStr ) const { wakeupStr = "Waking up to update the CollectionScheme: "; wakeupStr += mProcessCollectionScheme ? "Yes" : "No"; - wakeupStr += " and the DecoderManifest: "; + wakeupStr += ", the DecoderManifest: "; wakeupStr += mProcessDecoderManifest ? "Yes" : "No"; } -// Clears both enabled collectionScheme map and idle collectionScheme map -// removes all dataPair from mTimeLine except for CHECKIN -void -CollectionSchemeManager::cleanupCollectionSchemes() -{ - if ( mEnabledCollectionSchemeMap.empty() && mIdleCollectionSchemeMap.empty() ) - { - // already cleaned up - return; - } - mEnabledCollectionSchemeMap.clear(); - mIdleCollectionSchemeMap.clear(); - - // when cleaning up mTimeLine checkIn event needs to be preserved - TimeData saveTimeData = { { 0, 0 }, "" }; - while ( !mTimeLine.empty() ) - { - if ( mTimeLine.top().id == CHECKIN ) - { - saveTimeData = mTimeLine.top(); - } - mTimeLine.pop(); - } - if ( saveTimeData.time.monotonicTimeMs != 0 ) - { - mTimeLine.push( saveTimeData ); - } -} - void CollectionSchemeManager::doWork( void *data ) { CollectionSchemeManager *collectionSchemeManager = static_cast( data ); - bool enabledCollectionSchemeMapChanged = false; - // Set up timer for checkin messages - collectionSchemeManager->prepareCheckinTimer(); - // Retrieve collectionSchemeList and decoderManifest from persistent storage + // Retrieve data from persistent storage static_cast( collectionSchemeManager->retrieve( DataType::COLLECTION_SCHEME_LIST ) ); static_cast( collectionSchemeManager->retrieve( DataType::DECODER_MANIFEST ) ); + bool initialCheckinDocumentsUpdate = true; while ( true ) { + bool decoderManifestChanged = false; + bool enabledCollectionSchemeMapChanged = false; if ( collectionSchemeManager->mProcessDecoderManifest ) { collectionSchemeManager->mProcessDecoderManifest = false; TraceModule::get().sectionBegin( TraceSection::MANAGER_DECODER_BUILD ); if ( collectionSchemeManager->processDecoderManifest() ) { - enabledCollectionSchemeMapChanged = true; + decoderManifestChanged = true; } TraceModule::get().sectionEnd( TraceSection::MANAGER_DECODER_BUILD ); } @@ -195,43 +162,51 @@ CollectionSchemeManager::doWork( void *data ) { enabledCollectionSchemeMapChanged = true; } - if ( enabledCollectionSchemeMapChanged ) + + bool documentsChanged = decoderManifestChanged || enabledCollectionSchemeMapChanged; + + if ( documentsChanged || initialCheckinDocumentsUpdate ) + { + initialCheckinDocumentsUpdate = false; + collectionSchemeManager->updateCheckinDocuments(); + } + + if ( documentsChanged ) { TraceModule::get().sectionBegin( TraceSection::MANAGER_EXTRACTION ); + FWE_LOG_TRACE( "Start extraction at system time " + std::to_string( checkTime.systemTimeMs ) ); + auto inspectionMatrixOutput = std::make_shared(); TraceModule::get().sectionBegin( TraceSection::COLLECTION_SCHEME_CHANGE_TO_FIRST_DATA ); - FWE_LOG_TRACE( - "Start extraction because of changed active collection schemes at system time " + - std::to_string( checkTime.systemTimeMs ) ); - /* - * Extract InspectionMatrix from mEnabledCollectionSchemeMap - * - * input: mEnabledCollectionSchemeMap - * output: shared_ptr to InspectionMatrix - * - * Then, propagate inspection matrix to Inspection engine - */ - enabledCollectionSchemeMapChanged = false; - auto inspectionMatrixOutput = std::make_shared(); - collectionSchemeManager->inspectionMatrixExtractor( inspectionMatrixOutput ); - collectionSchemeManager->inspectionMatrixUpdater( inspectionMatrixOutput ); - /* - * extract decoder dictionary - * input: mDecoderManifest - * output: shared_ptr to decoderDictionary - * - * the propagate the output to Vehicle Data Consumers - */ + // Extract InspectionMatrix from mEnabledCollectionSchemeMap + collectionSchemeManager->updateActiveCollectionSchemeListeners(); + collectionSchemeManager->matrixExtractor( inspectionMatrixOutput ); + std::string enabled; + std::string idle; + collectionSchemeManager->printExistingCollectionSchemes( enabled, idle ); + FWE_LOG_INFO( "FWE activated collection schemes:" + enabled + " using decoder manifest:" + + collectionSchemeManager->mCurrentDecoderManifestID + " resulting in " + + std::to_string( inspectionMatrixOutput->conditions.size() ) + " inspection conditions" ); + + // Extract decoder dictionary std::map> decoderDictionaryMap; - collectionSchemeManager->decoderDictionaryExtractor( decoderDictionaryMap ); - // Publish decoder dictionaries update to all listeners + collectionSchemeManager->decoderDictionaryExtractor( decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + inspectionMatrixOutput +#endif + ); + + // Only notify the listeners after both have been extracted since the decoder dictionary + // extraction might have modified the inspection matrix. collectionSchemeManager->decoderDictionaryUpdater( decoderDictionaryMap ); - // coverity[check_return : SUPPRESS] + collectionSchemeManager->inspectionMatrixUpdater( inspectionMatrixOutput ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA // Update the Raw Buffer Config if ( collectionSchemeManager->mRawDataBufferManager != nullptr ) { + std::unordered_map updatedSignals; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA std::shared_ptr complexDataDictionary; auto decoderDictionary = decoderDictionaryMap.find( VehicleDataSourceProtocol::COMPLEX_DATA ); if ( decoderDictionary != decoderDictionaryMap.end() && decoderDictionary->second != nullptr ) @@ -243,9 +218,13 @@ CollectionSchemeManager::doWork( void *data ) FWE_LOG_WARN( "Could not cast dictionary to ComplexDataDecoderDictionary" ); } } - collectionSchemeManager->updateRawDataBufferConfig( complexDataDictionary ); - } + collectionSchemeManager->updateRawDataBufferConfigComplexSignals( complexDataDictionary, + updatedSignals ); #endif + FWE_LOG_INFO( "Updating raw buffer configuration for " + std::to_string( updatedSignals.size() ) + + " signals" ); + collectionSchemeManager->mRawDataBufferManager->updateConfig( updatedSignals ); + } auto canDecoderDictionaryPtr = std::dynamic_pointer_cast( decoderDictionaryMap[VehicleDataSourceProtocol::RAW_SOCKET] ); @@ -273,19 +252,11 @@ CollectionSchemeManager::doWork( void *data ) collectionSchemeManager->mCurrentDecoderManifestID + " resulting in decoding rules for " + std::to_string( decoderDictionaryMap.size() ) + " protocols. Decoder CAN channels: " + decoderCanChannels + " and OBD PIDs:" + obdPids ); - std::string enabled; - std::string idle; - collectionSchemeManager->printExistingCollectionSchemes( enabled, idle ); - // coverity[check_return : SUPPRESS] - FWE_LOG_INFO( "FWE activated collection schemes:" + enabled + " using decoder manifest:" + - collectionSchemeManager->mCurrentDecoderManifestID + " resulting in " + - std::to_string( inspectionMatrixOutput->conditions.size() ) + " inspection conditions" ); TraceModule::get().sectionEnd( TraceSection::MANAGER_EXTRACTION ); } /* * get next timePoint from the minHeap top * check if it is a valid timePoint, it can be obsoleted if start Time or stop Time gets updated - * It should be always valid because Checkin is default to be running all the time */ auto currentMonotonicTime = collectionSchemeManager->mClock->monotonicTimeSinceEpochMs(); if ( collectionSchemeManager->mTimeLine.empty() ) @@ -315,6 +286,27 @@ CollectionSchemeManager::doWork( void *data ) } } +void +CollectionSchemeManager::updateCheckinDocuments() +{ + // Create a list of active collectionSchemes and the current decoder manifest and send it to cloud + std::vector checkinMsg; + for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); it++ ) + { + checkinMsg.emplace_back( it->first ); + } + for ( auto it = mIdleCollectionSchemeMap.begin(); it != mIdleCollectionSchemeMap.end(); it++ ) + { + checkinMsg.emplace_back( it->first ); + } + if ( !mCurrentDecoderManifestID.empty() ) + { + checkinMsg.emplace_back( mCurrentDecoderManifestID ); + } + + mCheckinSender->onCheckinDocumentsChanged( checkinMsg ); +} + /* callback function */ void CollectionSchemeManager::onCollectionSchemeUpdate( const ICollectionSchemeListPtr &collectionSchemeList ) @@ -352,39 +344,6 @@ CollectionSchemeManager::updateAvailable() mDecoderManifestAvailable = false; } -/* - * checkinIntervalMsec - checkin interval in ms - * debounceIntervalMsec - debounce Interval in ms - */ -bool -CollectionSchemeManager::init( uint32_t checkinIntervalMsec, - const std::shared_ptr &schemaPersistencyPtr, - CANInterfaceIDTranslator &canIDTranslator -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::shared_ptr rawDataBufferManager -#endif -) -{ - mCANIDTranslator = canIDTranslator; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - mRawDataBufferManager = rawDataBufferManager; -#endif - FWE_LOG_TRACE( "CollectionSchemeManager initialised with a checkin interval of: " + - std::to_string( checkinIntervalMsec ) + " ms" ); - if ( checkinIntervalMsec > 0 ) - { - mCheckinIntervalInMsec = checkinIntervalMsec; - } - else - { - /* use default value when checkin interval is not set in configuration */ - mCheckinIntervalInMsec = DEFAULT_CHECKIN_INTERVAL_IN_MILLISECOND; - } - mSchemaPersistency = schemaPersistencyPtr; - return true; -} - bool CollectionSchemeManager::connect() { @@ -440,18 +399,7 @@ CollectionSchemeManager::processDecoderManifest() // store the new DM, update mCurrentDecoderManifestID mCurrentDecoderManifestID = mDecoderManifest->getID(); store( DataType::DECODER_MANIFEST ); - // when DM changes, check if we have collectionScheme loaded - if ( isCollectionSchemeLoaded() ) - { - // DM has changes, all existing collectionSchemes need to be cleared - cleanupCollectionSchemes(); - return false; - } - else - { - // collectionScheme maps are empty - return rebuildMapsandTimeLine( mClock->timeSinceEpoch() ); - } + return true; } /* @@ -520,22 +468,9 @@ CollectionSchemeManager::rebuildMapsandTimeLine( const TimePoint &currTime ) /* Separate collectionSchemes into Enabled and Idle bucket */ for ( auto const &collectionScheme : collectionSchemeList ) { - if ( collectionScheme->getDecoderManifestID() != mCurrentDecoderManifestID ) - { - // Encounters a collectionScheme that does not have matching DM - // Rebuild has to bail out. Call cleanupCollectionSchemes() before exiting. - FWE_LOG_TRACE( - "CollectionScheme does not have matching DM ID. Current DM ID: " + mCurrentDecoderManifestID + - " but collection scheme " + collectionScheme->getCollectionSchemeID() + " needs " + - collectionScheme->getDecoderManifestID() ); - - cleanupCollectionSchemes(); - return false; - } - // collectionScheme does not have matching DM, can't rebuild. Exit - Timestamp startTime = collectionScheme->getStartTime(); - Timestamp stopTime = collectionScheme->getExpiryTime(); - std::string id = collectionScheme->getCollectionSchemeID(); + auto startTime = collectionScheme->getStartTime(); + auto stopTime = collectionScheme->getExpiryTime(); + auto id = collectionScheme->getCollectionSchemeID(); if ( startTime > currTime.systemTimeMs ) { /* for idleCollectionSchemes, push both startTime and stopTime to timeLine */ @@ -551,8 +486,6 @@ CollectionSchemeManager::rebuildMapsandTimeLine( const TimePoint &currTime ) ret = true; } } - // Notify consumers of active collection schemes about new list - updateActiveSchemesListeners(); std::string enableStr; std::string idleStr; @@ -561,11 +494,11 @@ CollectionSchemeManager::rebuildMapsandTimeLine( const TimePoint &currTime ) return ret; } -std::vector +std::vector CollectionSchemeManager::getCollectionSchemeArns() { std::lock_guard lock( mSchemaUpdateMutex ); - std::vector collectionSchemeArns; + std::vector collectionSchemeArns; if ( mCollectionSchemeList != nullptr ) { for ( auto &collectionScheme : mCollectionSchemeList->getCollectionSchemes() ) @@ -590,7 +523,7 @@ bool CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) { bool ret = false; - std::unordered_set newCollectionSchemeIDs; + std::unordered_set newCollectionSchemeIDs; std::vector collectionSchemeList; if ( mCollectionSchemeList == nullptr ) @@ -601,16 +534,6 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) collectionSchemeList = mCollectionSchemeList->getCollectionSchemes(); for ( auto const &collectionScheme : collectionSchemeList ) { - if ( collectionScheme->getDecoderManifestID() != mCurrentDecoderManifestID ) - { - // Encounters a collectionScheme that does not have matching DM - // Rebuild has to bail out. Call cleanupCollectionSchemes() before exiting. - FWE_LOG_TRACE( "CollectionScheme does not have matching DM ID: " + mCurrentDecoderManifestID + " " + - collectionScheme->getDecoderManifestID() ); - - cleanupCollectionSchemes(); - return false; - } /* * Once collectionScheme has a matching DM, try to locate the collectionScheme in existing maps * using collectionScheme ID. @@ -628,7 +551,7 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) Timestamp startTime = collectionScheme->getStartTime(); Timestamp stopTime = collectionScheme->getExpiryTime(); - std::string id = collectionScheme->getCollectionSchemeID(); + auto id = collectionScheme->getCollectionSchemeID(); newCollectionSchemeIDs.insert( id ); auto itEnabled = mEnabledCollectionSchemeMap.find( id ); auto itIdle = mIdleCollectionSchemeMap.find( id ); @@ -646,11 +569,20 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) printEventLogMsg( completedStr, id, startTime, stopTime, currTime ); FWE_LOG_TRACE( completedStr ); } - else if ( stopTime != currCollectionScheme->getExpiryTime() ) + else { - /* StopTime changes on that collectionScheme, update with new CollectionScheme */ - mEnabledCollectionSchemeMap[id] = collectionScheme; - mTimeLine.push( { calculateMonotonicTime( currTime, stopTime ), id } ); + if ( stopTime != currCollectionScheme->getExpiryTime() ) + { + /* StopTime changes on that collectionScheme, update with new CollectionScheme */ + mEnabledCollectionSchemeMap[id] = collectionScheme; + mTimeLine.push( { calculateMonotonicTime( currTime, stopTime ), id } ); + } + + if ( *collectionScheme != *currCollectionScheme ) + { + mEnabledCollectionSchemeMap[id] = collectionScheme; + ret = true; + } } } else if ( itIdle != mIdleCollectionSchemeMap.end() ) @@ -679,6 +611,10 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) mTimeLine.push( { calculateMonotonicTime( currTime, startTime ), id } ); mTimeLine.push( { calculateMonotonicTime( currTime, stopTime ), id } ); } + else + { + mIdleCollectionSchemeMap[id] = collectionScheme; + } } else { @@ -738,8 +674,6 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) { FWE_LOG_TRACE( "Removing collectionSchemes missing from PI updates: " + removeStr ); } - // Notify consumers of active collection schemes about new list - updateActiveSchemesListeners(); std::string enableStr; std::string idleStr; @@ -755,7 +689,6 @@ CollectionSchemeManager::updateMapsandTimeLine( const TimePoint &currTime ) * If not, simply exit, return false; * 2. Otherwise, * get topTime and topCollectionSchemeID from top of MINheap, - * if it is checkin event, simply send out checkin, this is false case; * if collectionScheme in Enabled Map, and stopTime equal to topTime, time to disable this collectionScheme, this * is a true case; else if CollectionScheme in idle map, and startTime equals to topTime, time to enable this * collectionScheme, this is a true case; for the rest of the cases, all false; @@ -780,52 +713,9 @@ CollectionSchemeManager::checkTimeLine( const TimePoint &currTime ) while ( !mTimeLine.empty() ) { const auto &topPair = mTimeLine.top(); - const std::string &topCollectionSchemeID = topPair.id; - const TimePoint &topTime = topPair.time; - if ( topCollectionSchemeID == CHECKIN ) - { - // for checkin, we are about to - // either serve current checkin event, and move on to search for next timePoint to set up timer; - // or we find current checkin for setting up next timer, then we are done here; - if ( currTime.monotonicTimeMs < topTime.monotonicTimeMs ) - { - // Successfully locate next checkin as timePoint to set up timer - // time to exit - break; - } - // Try to send the checkin message. - // If it succeeds, we will schedule the next checkin cycle using the provided interval. - // If it does not succeed ( e.g. no offboardconnectivity), we schedule for a retry immediately. - bool checkinSuccess = sendCheckin(); - // Now schedule based on the return code - if ( checkinSuccess ) - { - if ( mCheckinIntervalInMsec > 0 ) - { - TimePoint nextCheckinTime = { currTime.systemTimeMs + mCheckinIntervalInMsec, - currTime.monotonicTimeMs + mCheckinIntervalInMsec }; - mTimeLine.push( { nextCheckinTime, CHECKIN } ); - } - // else, no checkin message is scheduled. - } - else - { - // Schedule with for a quick retry - // Calculate the minimum retry interval - uint64_t minimumCheckinInterval = - std::min( static_cast( RETRY_CHECKIN_INTERVAL_IN_MILLISECOND ), mCheckinIntervalInMsec ); - TimePoint nextCheckinTime = { currTime.systemTimeMs + minimumCheckinInterval, - currTime.monotonicTimeMs + minimumCheckinInterval }; - mTimeLine.push( { nextCheckinTime, CHECKIN } ); - } - - // after sending checkin, the work on this dataPair is done, move to next dataPair - // to look for next valid timePoint to set up timer - mTimeLine.pop(); - continue; - } + const auto &topCollectionSchemeID = topPair.id; + const auto &topTime = topPair.time; - // in case of non-checkin // first find topCollectionSchemeID in mEnabledCollectionSchemeMap then mIdleCollectionSchemeMap // if we find a match in collectionScheme ID, check further if topTime matches this collectionScheme's // start/stop time @@ -929,15 +819,42 @@ CollectionSchemeManager::checkTimeLine( const TimePoint &currTime ) return ret; } +bool +CollectionSchemeManager::isCollectionSchemesInSyncWithDm() +{ + for ( const auto &collectionScheme : mEnabledCollectionSchemeMap ) + { + if ( collectionScheme.second->getDecoderManifestID() != mCurrentDecoderManifestID ) + { + FWE_LOG_INFO( "Decoder manifest out of sync: " + collectionScheme.second->getDecoderManifestID() + " vs. " + + mCurrentDecoderManifestID ); + return false; + } + } + for ( const auto &collectionScheme : mIdleCollectionSchemeMap ) + { + if ( collectionScheme.second->getDecoderManifestID() != mCurrentDecoderManifestID ) + { + FWE_LOG_INFO( "Decoder manifest out of sync: " + collectionScheme.second->getDecoderManifestID() + " vs. " + + mCurrentDecoderManifestID ); + return false; + } + } + return true; +} + void -CollectionSchemeManager::updateActiveSchemesListeners() +CollectionSchemeManager::updateActiveCollectionSchemeListeners() { // Create vector of active collection schemes to notify interested components about new schemes auto activeCollectionSchemesOutput = std::make_shared(); - for ( const auto &enabledCollectionScheme : mEnabledCollectionSchemeMap ) + if ( isCollectionSchemesInSyncWithDm() ) { - activeCollectionSchemesOutput->activeCollectionSchemes.push_back( enabledCollectionScheme.second ); + for ( const auto &enabledCollectionScheme : mEnabledCollectionSchemeMap ) + { + activeCollectionSchemesOutput->activeCollectionSchemes.push_back( enabledCollectionScheme.second ); + } } mCollectionSchemeListChangeListeners.notify( activeCollectionSchemesOutput ); diff --git a/src/CollectionSchemeManager.h b/src/CollectionSchemeManager.h index 75810c09..c4aabdf5 100644 --- a/src/CollectionSchemeManager.h +++ b/src/CollectionSchemeManager.h @@ -5,15 +5,16 @@ #include "CANInterfaceIDTranslator.h" #include "CacheAndPersist.h" +#include "CheckinSender.h" #include "Clock.h" #include "ClockHandler.h" #include "CollectionInspectionAPITypes.h" +#include "ICollectionScheme.h" #include "ICollectionSchemeList.h" -#include "ICollectionSchemeManager.h" #include "IDecoderDictionary.h" #include "IDecoderManifest.h" #include "Listener.h" -#include "SchemaListener.h" +#include "RawDataManager.h" #include "Signal.h" #include "SignalTypes.h" #include "Thread.h" @@ -31,7 +32,7 @@ #ifdef FWE_FEATURE_VISION_SYSTEM_DATA #include "MessageTypes.h" -#include "RawDataManager.h" +#include #endif namespace Aws @@ -39,13 +40,11 @@ namespace Aws namespace IoTFleetWise { -using SchemaListenerPtr = std::shared_ptr; - /* TimeData is used in mTimeline, the second parameter in the pair is a CollectionScheme ID */ struct TimeData { TimePoint time; - std::string id; + SyncID id; bool operator>( const TimeData &other ) const @@ -67,8 +66,7 @@ struct TimeData request. * 7. Notify other components about currently Enabled CollectionSchemes. */ - -class CollectionSchemeManager : public ICollectionSchemeManager +class CollectionSchemeManager // NOLINT(clang-analyzer-optin.performance.Padding) { public: /** @@ -99,38 +97,25 @@ class CollectionSchemeManager : public ICollectionSchemeManager using OnCollectionSchemeListChangeCallback = std::function &activeCollectionSchemes )>; - CollectionSchemeManager() = default; - - CollectionSchemeManager( std::string dm_id ); - - CollectionSchemeManager( std::string dm_id, - std::map mapEnabled, - std::map mapIdle ); + CollectionSchemeManager( + std::shared_ptr + schemaPersistencyPtr, /**< shared pointer to collectionSchemePersistency object */ + CANInterfaceIDTranslator &canIDTranslator, /**< canIDTranslator used to translate the cloud used Interface + ID to the the internal channel id */ + std::shared_ptr + checkinSender, /**< the checkin sender that needs to be updated with the current documents */ + std::shared_ptr rawDataBufferManager = + nullptr /**< rawDataBufferManager Optional manager to handle raw data. If not given, raw data + collection will be disabled */ + ); - ~CollectionSchemeManager() override; + ~CollectionSchemeManager(); CollectionSchemeManager( const CollectionSchemeManager & ) = delete; CollectionSchemeManager &operator=( const CollectionSchemeManager & ) = delete; CollectionSchemeManager( CollectionSchemeManager && ) = delete; CollectionSchemeManager &operator=( CollectionSchemeManager && ) = delete; - /** - * @brief Initializes collectionScheme management session. - * - * @return True if successful. False otherwise. - */ - bool init( uint32_t checkinIntervalMsec, /**< checkin message interval in millisecond */ - const std::shared_ptr - &schemaPersistencyPtr, /**< shared pointer to collectionSchemePersistency object */ - CANInterfaceIDTranslator &canIDTranslator /**< canIDTranslator used to translate the cloud used Interface - ID to the the internal channel id */ -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::shared_ptr rawDataBufferManager = - nullptr /**< rawDataBufferManager Optional manager to handle raw data. If not given, raw data - collection will be disabled */ -#endif - ); /** * @brief Sets up connection with CollectionScheme Ingestion and start main thread. * @return True if successful. False otherwise. @@ -158,7 +143,7 @@ class CollectionSchemeManager : public ICollectionSchemeManager * A lock in the function is applied to handle the race condition between AwdIoT context and PM context. * */ - void onCollectionSchemeUpdate( const ICollectionSchemeListPtr &collectionSchemeList ) override; + void onCollectionSchemeUpdate( const ICollectionSchemeListPtr &collectionSchemeList ); /** * @brief callback for CollectionScheme Ingestion to send update of IDecoderManifest @@ -170,25 +155,13 @@ class CollectionSchemeManager : public ICollectionSchemeManager * A lock in the function is applied to handle the race condition between AwdIoT context and PM context. */ - void onDecoderManifestUpdate( const IDecoderManifestPtr &decoderManifest ) override; - - /** - * @brief Used by the bootstrap to set the SchemaListener pointer to allow CollectionScheme Management to send data - * to CollectionScheme Ingestion - * - * @param collectionSchemeIngestionListenerPtr - */ - inline void - setSchemaListenerPtr( const SchemaListenerPtr collectionSchemeIngestionListenerPtr ) - { - mSchemaListenerPtr = collectionSchemeIngestionListenerPtr; - } + void onDecoderManifestUpdate( const IDecoderManifestPtr &decoderManifest ); /** * @brief Returns the current list of collection scheme ARNs * @return List of collection scheme ARNs */ - std::vector getCollectionSchemeArns(); + std::vector getCollectionSchemeArns(); /** * @brief Subscribe to changes in the active decoder dictionary @@ -258,7 +231,7 @@ class CollectionSchemeManager : public ICollectionSchemeManager * @param currTime time when main thread wakes up */ static void printEventLogMsg( std::string &msg, - const std::string &id, + const SyncID &id, const Timestamp &startTime, const Timestamp &stopTime, const TimePoint &currTime ); @@ -279,47 +252,61 @@ class CollectionSchemeManager : public ICollectionSchemeManager */ void printWakeupStatus( std::string &wakeupStr ) const; - /** - * @brief clean up collectionScheme maps and timeline - * - */ - void cleanupCollectionSchemes(); - /** * @brief Fill up the fields in ConditionWithCollectedData - * To be called by inspectionMatrixExtractor + * To be called by matrixExtractor * * @param collectionScheme the collectionScheme where the info is retrieved * @param conditionData object ConditionWithCollectedData to be filled */ void addConditionData( const ICollectionSchemePtr &collectionScheme, ConditionWithCollectedData &conditionData ); - /** - * @brief initialize in mTimeLine to send checkin message - */ - void prepareCheckinTimer(); - /** * @brief checks if there is any enabled or idle collectionScheme in the system * returns true when there is */ bool isCollectionSchemeLoaded(); + bool isCollectionSchemesInSyncWithDm(); + + void extractCondition( const std::shared_ptr &inspectionMatrix, + const ICollectionSchemePtr &collectionScheme, + std::vector &nodes, + std::map &nodeToIndexMap, + uint32_t &index, + const ExpressionNode *initialNode ); + protected: - bool rebuildMapsandTimeLine( const TimePoint &currTime ) override; + bool rebuildMapsandTimeLine( const TimePoint &currTime ); - bool updateMapsandTimeLine( const TimePoint &currTime ) override; + bool updateMapsandTimeLine( const TimePoint &currTime ); - bool checkTimeLine( const TimePoint &currTime ) override; + bool checkTimeLine( const TimePoint &currTime ); /** * @brief This function extract the decoder dictionary from decoder manifest and polices - * - * @param decoderDictionaryMap pass reference of the map of decoder dictionary. This map contains dictionaries for - * different network types */ void decoderDictionaryExtractor( - std::map> &decoderDictionaryMap ); + std::map> + &decoderDictionaryMap /**< pass reference of the map of decoder dictionary. This map contains dictionaries + for different network types */ +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + std::shared_ptr inspectionMatrix /**< the inspection matrix that will be updated with the + right signal types for partial signals */ +#endif + ); + + void addSignalToDecoderDictionaryMap( + SignalID signalId, + std::map> &decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + std::unordered_map &partialSignalTypes, + SignalID topLevelSignalId = INVALID_SIGNAL_ID, + SignalPath signalPath = SignalPath() +#endif + ); #ifdef FWE_FEATURE_VISION_SYSTEM_DATA /** @@ -330,18 +317,23 @@ class CollectionSchemeManager : public ICollectionSchemeManager * @param partialSignalID the ID that should be used for a partial signal. Only used if signalPath is not empty * @param signalPath if not empty this signal is a partialSignal and partialSignalID will be use * @param complexSignalRootType the root complex type of this signal + * @param partialSignalTypes the map that will be updated with the signal types for partial signals */ void putComplexSignalInDictionary( ComplexDataMessageFormat &complexSignal, SignalID signalID, PartialSignalID partialSignalID, SignalPath &signalPath, - ComplexDataTypeId complexSignalRootType ); + ComplexDataTypeId complexSignalRootType, + std::unordered_map &partialSignalTypes ); /** - * @brief Fills up and creates the BufferConfig and sends it to the Raw Buffer Manager + * @brief Fills up and creates the BufferConfig with complex signals + * @param complexDataDecoderDictionary current complex data decoder dict + * @param updatedSignals map of the signals that will be updated by Raw Buffer Manager */ - void updateRawDataBufferConfig( - std::shared_ptr complexDataDecoderDictionary ); + void updateRawDataBufferConfigComplexSignals( + std::shared_ptr complexDataDecoderDictionary, + std::unordered_map &updatedSignals ); #endif /** @@ -354,30 +346,23 @@ class CollectionSchemeManager : public ICollectionSchemeManager void decoderDictionaryUpdater( std::map> &decoderDictionaryMap ); - void inspectionMatrixExtractor( const std::shared_ptr &inspectionMatrix ) override; + void matrixExtractor( const std::shared_ptr &inspectionMatrix ); - void inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) override; + void inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ); - bool retrieve( DataType retrieveType ) override; + bool retrieve( DataType retrieveType ); - void store( DataType storeType ) override; + void store( DataType storeType ); - bool processDecoderManifest() override; + bool processDecoderManifest(); - bool processCollectionScheme() override; + bool processCollectionScheme(); - bool sendCheckin() override; + void updateCheckinDocuments(); - void updateAvailable() override; + void updateAvailable(); private: - // default checkin interval set to 5 mins - static constexpr int DEFAULT_CHECKIN_INTERVAL_IN_MILLISECOND = 300000; - // Checkin retry interval. Used issue checkins to the cloud as soon as possible, set to 5 seconds - static constexpr int RETRY_CHECKIN_INTERVAL_IN_MILLISECOND = 5000; - // checkIn ID in parallel to collectionScheme IDs - static const std::string CHECKIN; - Thread mThread; // Atomic flag to signal the state of main thread. If true, we should stop std::atomic mShouldStop{ false }; @@ -388,38 +373,40 @@ class CollectionSchemeManager : public ICollectionSchemeManager Signal mWait; std::shared_ptr mClock = ClockHandler::getClock(); - // Shared pointer to a SchemaListener Object allow CollectionSchemeManagement to send data to Schema - SchemaListenerPtr mSchemaListenerPtr; + std::shared_ptr mCheckinSender; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA // Shared pointer to a Raw Data Buffer Manager Object allow CollectionSchemeManagement to send BufferConfig to // the Manager std::shared_ptr mRawDataBufferManager; -#endif - - // Idle collectionScheme collection - std::map mIdleCollectionSchemeMap; - - // Enabled collectionScheme collection - std::map mEnabledCollectionSchemeMap; // Builds vector of ActiveCollectionSchemes and notifies listeners about the update - void updateActiveSchemesListeners(); - - // ID for the decoder manifest currently in use - std::string mCurrentDecoderManifestID; - - // Time interval in ms to send checkin message - uint64_t mCheckinIntervalInMsec{ DEFAULT_CHECKIN_INTERVAL_IN_MILLISECOND }; + void updateActiveCollectionSchemeListeners(); // Get the Signal Type from DM inline SignalType getSignalType( const SignalID signalID ) { +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // For internal signals we won't find the type in the decoder manifest. The type will be + // determined later after both collection schemes and decoder manifest are processed. + if ( ( signalID & INTERNAL_SIGNAL_ID_BITMASK ) != 0 ) + { + return SignalType::UNKNOWN; + } +#endif return mDecoderManifest->getSignalType( signalID ); } protected: + // Idle collectionScheme collection + std::map mIdleCollectionSchemeMap; + + // Enabled collectionScheme collection + std::map mEnabledCollectionSchemeMap; + + // ID for the decoder manifest currently in use + SyncID mCurrentDecoderManifestID; + /* * PM Local storage of CollectionSchemeList and mDecoderManifest so that PM can work on these objects * out of critical section diff --git a/src/CustomDataSource.cpp b/src/CustomDataSource.cpp index b625bcb9..3873fc2e 100644 --- a/src/CustomDataSource.cpp +++ b/src/CustomDataSource.cpp @@ -4,6 +4,8 @@ #include "CustomDataSource.h" #include "LoggingModule.h" #include "Timer.h" +#include +#include #include #include @@ -39,6 +41,11 @@ CustomDataSource::start() const auto name = getThreadName(); if ( name != nullptr ) { + if ( strnlen( name, 16 ) > 15 ) + { + FWE_LOG_WARN( "Thread name '" + std::string( name ) + + "' is larger than 15 chars. Setting the thread name will likely fail." ); + } mThread.setThreadName( name ); } } diff --git a/src/DataSenderIonWriter.cpp b/src/DataSenderIonWriter.cpp index c0ee7202..7d2ecce8 100644 --- a/src/DataSenderIonWriter.cpp +++ b/src/DataSenderIonWriter.cpp @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + #include "DataSenderIonWriter.h" #include "EventTypes.h" #include "LoggingModule.h" @@ -97,6 +100,8 @@ class IonFileGenerator : public std::streambuf return mIonWriteBuffer[0]; } + // coverity[autosar_cpp14_m3_9_1_violation] false-positive, redeclaration is compatible + // coverity[misra_cpp_2008_rule_3_9_1_violation] same std::streampos seekpos( std::streampos sp, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ) override { @@ -362,7 +367,7 @@ class IonFileGenerator : public std::streambuf // coverity[misra_cpp_2008_rule_5_0_9_violation] same // coverity[cert_ctr54_cpp_violation] same mWrittenBytesInIonWriteBuffer = - static_cast( stream->curr - static_cast( &( *mIonWriteBuffer.begin() ) ) ); + static_cast( stream->curr - static_cast( &( *mIonWriteBuffer.begin() ) ) ); } if ( stream->curr == stream->limit ) { // only increase buffer if necessary @@ -523,8 +528,8 @@ class IonFileGenerator : public std::streambuf Timestamp mTriggerTime; EventID mEventId; - std::string mDecoderManifestId; - std::string mCollectionSchemeId; + SyncID mDecoderManifestId; + SyncID mCollectionSchemeId; std::string mVehicleId; // Metadata: @@ -551,8 +556,8 @@ class IonStreambufBuilder : public StreambufBuilder std::string vehicleId, Timestamp triggerTime, EventID eventId, - std::string decoderManifestId, - std::string collectionSchemeId ) + SyncID decoderManifestId, + SyncID collectionSchemeId ) : mRawDataBufferManager( std::move( rawDataBufferManager ) ) , mVehicleId( std::move( vehicleId ) ) , mTriggerTime( triggerTime ) @@ -590,8 +595,8 @@ class IonStreambufBuilder : public StreambufBuilder std::string mVehicleId; Timestamp mTriggerTime; EventID mEventId; - std::string mDecoderManifestId; - std::string mCollectionSchemeId; + SyncID mDecoderManifestId; + SyncID mCollectionSchemeId; std::vector mFramesToSendOut; }; @@ -643,7 +648,7 @@ DataSenderIonWriter::onChangeOfActiveDictionary( ConstDecoderDictionaryConstPtr DataSenderIonWriter::~DataSenderIonWriter() = default; void -DataSenderIonWriter::setupVehicleData( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeData ) +DataSenderIonWriter::setupVehicleData( std::shared_ptr triggeredVisionSystemData ) { mEstimatedBytesInCurrentStream = ESTIMATED_SERIALIZED_EVENT_METADATA_BYTES; // reset and start new if ( mRawDataBufferManager != nullptr ) @@ -652,10 +657,10 @@ DataSenderIonWriter::setupVehicleData( const TriggeredCollectionSchemeDataPtr &t mCurrentStreamBuilder = std::make_unique( mRawDataBufferManager, mVehicleId, - triggeredCollectionSchemeData->triggerTime, - triggeredCollectionSchemeData->eventID, - triggeredCollectionSchemeData->metadata.decoderID, - triggeredCollectionSchemeData->metadata.collectionSchemeID ); + triggeredVisionSystemData->triggerTime, + triggeredVisionSystemData->eventID, + triggeredVisionSystemData->metadata.decoderID, + triggeredVisionSystemData->metadata.collectionSchemeID ); } else { @@ -704,7 +709,7 @@ void DataSenderIonWriter::append( const CollectedSignal &signal ) { // Currently ION file only supports raw data - if ( ( signal.value.type == SignalType::RAW_DATA_BUFFER_HANDLE ) && ( mCurrentStreamBuilder != nullptr ) ) + if ( ( signal.value.type == SignalType::COMPLEX_SIGNAL ) && ( mCurrentStreamBuilder != nullptr ) ) { if ( mRawDataBufferManager != nullptr ) { diff --git a/src/DataSenderIonWriter.h b/src/DataSenderIonWriter.h index 5f8cd1d6..e61b6d39 100644 --- a/src/DataSenderIonWriter.h +++ b/src/DataSenderIonWriter.h @@ -59,10 +59,9 @@ class DataSenderIonWriter /** * @brief Starts a new set of data and a fresh stream * - * @param triggeredCollectionSchemeData pointer to the collected data and metadata - * to be sent to cloud + * @param triggeredVisionSystemData pointer to the collected data and metadata to be sent to cloud */ - virtual void setupVehicleData( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeData ); + virtual void setupVehicleData( std::shared_ptr triggeredVisionSystemData ); /** * @brief Hand over stream builder. After this to start a new stream setupVehicleData has to be called. @@ -78,7 +77,7 @@ class DataSenderIonWriter /** * @brief Appends the decoded raw frame handle to the stream generator * - * @param signal only type SignalType::RAW_DATA_BUFFER_HANDLE will be accepted + * @param signal only type SignalType::COMPLEX_SIGNAL will be accepted */ virtual void append( const CollectedSignal &signal ); diff --git a/src/DataSenderManager.cpp b/src/DataSenderManager.cpp index 477fd622..9a7f8bb1 100644 --- a/src/DataSenderManager.cpp +++ b/src/DataSenderManager.cpp @@ -4,367 +4,162 @@ #include "DataSenderManager.h" #include "CacheAndPersist.h" #include "LoggingModule.h" -#include "OBDDataTypes.h" -#include "TraceModule.h" -#include +#include +#include +#include +#include #include -#include +#include #include -#include - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "S3Sender.h" -#include "SignalTypes.h" -#include "StreambufBuilder.h" -#endif namespace Aws { namespace IoTFleetWise { -namespace -{ -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -constexpr char DEFAULT_KEY_SUFFIX[] = ".10n"; // Ion is the only supported format -#endif -} // namespace - -DataSenderManager::DataSenderManager( std::shared_ptr mqttSender, - std::shared_ptr payloadManager, - CANInterfaceIDTranslator &canIDTranslator, - unsigned transmitThreshold -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::shared_ptr s3Sender, - std::shared_ptr ionWriter, - std::string vehicleName -#endif - ) - : mMQTTSender( std::move( mqttSender ) ) +DataSenderManager::DataSenderManager( std::unordered_map> dataSenders, + std::shared_ptr mqttSender, + std::shared_ptr payloadManager ) + : mDataSenders( std::move( dataSenders ) ) + , mMQTTSender( std::move( mqttSender ) ) , mPayloadManager( std::move( payloadManager ) ) - , mProtoWriter( canIDTranslator ) -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , mIonWriter( std::move( ionWriter ) ) - , mS3Sender{ std::move( s3Sender ) } - , mVehicleName( std::move( vehicleName ) ) -#endif { - mTransmitThreshold = ( transmitThreshold > 0U ) ? transmitThreshold : UINT_MAX; } void -DataSenderManager::processCollectedData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::function reportUploadCallback -#endif -) +DataSenderManager::processData( std::shared_ptr data ) { - if ( triggeredCollectionSchemeDataPtr == nullptr ) + if ( data == nullptr ) { FWE_LOG_WARN( "Nothing to send as the input is empty" ); return; } - - setCollectionSchemeParameters( triggeredCollectionSchemeDataPtr ); - - transformTelemetryDataToProto( triggeredCollectionSchemeDataPtr ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - transformVisionSystemDataToIon( triggeredCollectionSchemeDataPtr, reportUploadCallback ); -#endif -} - -void -DataSenderManager::setCollectionSchemeParameters( - const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr ) -{ - mCollectionSchemeParams.persist = triggeredCollectionSchemeDataPtr->metadata.persist; - mCollectionSchemeParams.compression = triggeredCollectionSchemeDataPtr->metadata.compress; - mCollectionSchemeParams.priority = triggeredCollectionSchemeDataPtr->metadata.priority; - mCollectionSchemeParams.eventID = triggeredCollectionSchemeDataPtr->eventID; - mCollectionSchemeParams.triggerTime = triggeredCollectionSchemeDataPtr->triggerTime; - mCollectionSchemeID = triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID; -} - -void -DataSenderManager::transformTelemetryDataToProto( - const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr ) -{ - // Clear old data and setup metadata - mProtoWriter.setupVehicleData( triggeredCollectionSchemeDataPtr, mCollectionSchemeParams.eventID ); - - // Iterate through all the signals and add to the protobuf - for ( const auto &signal : triggeredCollectionSchemeDataPtr->signals ) + auto sender = mDataSenders.find( data->getDataType() ); + if ( sender == mDataSenders.end() ) { -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - // Filter out the raw data and internal signals - if ( ( signal.value.type != SignalType::RAW_DATA_BUFFER_HANDLE ) && - ( ( signal.signalID & INTERNAL_SIGNAL_ID_BITMASK ) == 0 ) ) -#endif - { - appendMessageToProto( triggeredCollectionSchemeDataPtr, signal ); - } - } - - // Iterate through all the raw CAN frames and add to the protobuf - for ( const auto &canFrame : triggeredCollectionSchemeDataPtr->canFrames ) - { - appendMessageToProto( triggeredCollectionSchemeDataPtr, canFrame ); + FWE_LOG_ERROR( "No sender configured for data type: " + + std::to_string( static_cast( data->getDataType() ) ) ); + return; } - // Add DTC info to the payload - if ( triggeredCollectionSchemeDataPtr->mDTCInfo.hasItems() ) - { - mProtoWriter.setupDTCInfo( triggeredCollectionSchemeDataPtr->mDTCInfo ); - const auto &dtcCodes = triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes; - - // Iterate through all the DTC codes and add to the protobuf - for ( const auto &dtc : dtcCodes ) + sender->second->processData( data, [this]( bool success, std::shared_ptr dataToPersist ) { + if ( success ) { - appendMessageToProto( triggeredCollectionSchemeDataPtr, dtc ); + FWE_LOG_TRACE( "Data successfully sent" ); } - } - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - for ( const auto &object : triggeredCollectionSchemeDataPtr->uploadedS3Objects ) - { - appendMessageToProto( triggeredCollectionSchemeDataPtr, object ); - } -#endif - - // Serialize and transmit any remaining messages - if ( mProtoWriter.getVehicleDataMsgCount() >= 1U ) - { - FWE_LOG_TRACE( "Queuing message for upload" ); - uploadProto(); - } -} - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -void -DataSenderManager::onChangeCollectionSchemeList( - const std::shared_ptr &activeCollectionSchemes ) -{ - FWE_LOG_INFO( "New active collection scheme list was handed over to Data Sender" ); - mActiveCollectionSchemes = activeCollectionSchemes; -} - -S3UploadMetadata -DataSenderManager::getS3UploadMetadataForCollectionScheme( const std::string &collectionSchemeID ) -{ - if ( mActiveCollectionSchemes != nullptr ) - { - for ( const auto &scheme : mActiveCollectionSchemes->activeCollectionSchemes ) + else if ( dataToPersist != nullptr ) { - if ( scheme->getCollectionSchemeID() == collectionSchemeID ) + FWE_LOG_ERROR( "Failed to send data, persisting it" ); + if ( mPayloadManager != nullptr ) { - return scheme->getS3UploadMetadata(); + auto dataVariant = dataToPersist->getData(); + if ( dataVariant.type() == typeid( std::shared_ptr ) ) + { + auto rawData = boost::get>( dataVariant ); + + Json::Value metadata; + metadata["type"] = senderDataTypeToString( dataToPersist->getDataType() ); + Json::Value payloadSpecificMetadata = dataToPersist->getMetadata(); + payloadSpecificMetadata["filename"] = dataToPersist->getFilename(); + payloadSpecificMetadata["payloadSize"] = static_cast( rawData->size() ); + metadata["payload"] = payloadSpecificMetadata; + + mPayloadManager->storeData( reinterpret_cast( rawData->data() ), + rawData->size(), + metadata, + dataToPersist->getFilename() ); + } + else if ( dataVariant.type() == typeid( std::shared_ptr ) ) + { + auto rawData = boost::get>( dataVariant ); + mPayloadManager->storeData( *rawData, dataToPersist->getMetadata(), dataToPersist->getFilename() ); + } } } - } - return S3UploadMetadata(); + else + { + FWE_LOG_ERROR( + "Failed to send data, but persistency is not enabled for this type of data. Discarding it." ); + } + } ); } void -DataSenderManager::transformVisionSystemDataToIon( - const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr, - std::function uploadedDataCallback ) +DataSenderManager::checkAndSendRetrievedData() { - if ( triggeredCollectionSchemeDataPtr->signals.empty() ) - { - return; - } - if ( mS3Sender == nullptr ) + // Retrieve the metadata from persistency library + Json::Value files; + if ( mPayloadManager == nullptr ) { - FWE_LOG_ERROR( "Can not send data to S3 as S3Sender is not initalized. Please make sure config parameters in " - "section credentialsProvider are correct" ); return; } - if ( mIonWriter == nullptr ) + ErrorCode status = mPayloadManager->retrievePayloadMetadata( files ); + + if ( status != ErrorCode::SUCCESS ) { - FWE_LOG_WARN( "IonWriter is not set for the upload to S3" ); + FWE_LOG_ERROR( "Payload Metadata Retrieval Failed" ); return; } - bool rawDataAvailableToSend = false; - bool vehicleDataIsSet = false; - // Append signals with raw data to Ion file - for ( const auto &signal : triggeredCollectionSchemeDataPtr->signals ) + FWE_LOG_TRACE( "Number of Payloads to transmit : " + std::to_string( files.size() ) ); + for ( const auto &item : files ) { - if ( signal.value.type != SignalType::RAW_DATA_BUFFER_HANDLE ) + auto payloadType = SenderDataType::TELEMETRY; + auto typeString = item["type"].asString(); + Json::Value payloadMetadata = item; + if ( !typeString.empty() ) { - continue; + payloadMetadata = item["payload"]; + if ( !stringToSenderDataType( typeString, payloadType ) ) + { + FWE_LOG_WARN( "Ignoring unsupported persisted data type: " + typeString ) + continue; + } } - rawDataAvailableToSend = true; - if ( !vehicleDataIsSet ) + else { - vehicleDataIsSet = true; - // Setup the next Ion file data only once - mIonWriter->setupVehicleData( triggeredCollectionSchemeDataPtr ); + FWE_LOG_TRACE( "Found legacy metadata. Assuming data type is Telemetry." ) } - mIonWriter->append( signal ); - } - if ( !rawDataAvailableToSend ) - { - return; - } - - auto s3UploadMetadata = - getS3UploadMetadataForCollectionScheme( triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID ); - if ( s3UploadMetadata == S3UploadMetadata() ) - { - FWE_LOG_WARN( "Collection scheme " + triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID + - " no longer active" ); - return; - } - // Get stream for the Ion file and upload it with S3 sender - auto streambufBuilder = mIonWriter->getStreambufBuilder(); - - std::string objectKey = s3UploadMetadata.prefix + std::to_string( triggeredCollectionSchemeDataPtr->eventID ) + - "-" + std::to_string( triggeredCollectionSchemeDataPtr->triggerTime ) + - &DEFAULT_KEY_SUFFIX[0]; - auto resultCallback = [objectKey, triggeredCollectionSchemeDataPtr, uploadedDataCallback]( bool success ) -> void { - if ( !success ) + auto sender = mDataSenders.find( payloadType ); + if ( sender == mDataSenders.end() ) { - return; + FWE_LOG_ERROR( "No sender configured for persisted data type: " + + std::to_string( static_cast( payloadType ) ) ); + continue; } - auto collectedData = std::make_shared(); - collectedData->metadata = triggeredCollectionSchemeDataPtr->metadata; - collectedData->eventID = triggeredCollectionSchemeDataPtr->eventID; - collectedData->triggerTime = triggeredCollectionSchemeDataPtr->triggerTime; - collectedData->uploadedS3Objects.push_back( UploadedS3Object{ objectKey, UploadedS3ObjectDataFormat::Cdr } ); - uploadedDataCallback( collectedData ); - }; - mS3Sender->sendStream( std::move( streambufBuilder ), s3UploadMetadata, objectKey, resultCallback ); -} -#endif -bool -DataSenderManager::serialize( std::string &output ) -{ - // Note: a class member is used to store the serialized proto output to avoid heap fragmentation - if ( !mProtoWriter.serializeVehicleData( &output ) ) - { - FWE_LOG_ERROR( "Serialization failed" ); - return false; - } - return true; -} - -bool -DataSenderManager::compress( std::string &input ) -{ - if ( mCollectionSchemeParams.compression ) - { - FWE_LOG_TRACE( "Compress the payload before transmitting since compression flag is true" ); - if ( snappy::Compress( input.data(), input.size(), &mCompressedProtoOutput ) == 0U ) + // Retrieve the payload as stream so that it can be lazily read by the sender + std::string filename = payloadMetadata["filename"].asString(); + size_t payloadSize = sizeof( size_t ) >= sizeof( uint64_t ) ? payloadMetadata["payloadSize"].asUInt64() + : payloadMetadata["payloadSize"].asUInt(); + std::ifstream payload; + if ( mPayloadManager->retrievePayloadLazily( payload, filename ) != ErrorCode::SUCCESS ) { - FWE_LOG_TRACE( "Error in compressing the payload" ); - return false; + continue; } - } - return true; -} -ConnectivityError -DataSenderManager::send( const std::uint8_t *data, size_t size, std::shared_ptr sender ) -{ - if ( sender == nullptr ) - { - FWE_LOG_ERROR( "No sender provided" ); - return ConnectivityError::NotConfigured; - } - - ConnectivityError ret = sender->sendBuffer( data, size, mCollectionSchemeParams ); - if ( ret != ConnectivityError::Success ) - { - FWE_LOG_ERROR( "Failed to send vehicle data with error: " + std::to_string( static_cast( ret ) ) ); - } - else - { - TraceModule::get().sectionEnd( TraceSection::COLLECTION_SCHEME_CHANGE_TO_FIRST_DATA ); - TraceModule::get().incrementVariable( TraceVariable::MQTT_SIGNAL_MESSAGES_SENT_OUT ); - FWE_LOG_INFO( "A Payload of size: " + std::to_string( size ) + " bytes has been uploaded" ); - } - return ret; -} - -void -DataSenderManager::uploadProto() -{ - if ( !serialize( mProtoOutput ) ) - { - FWE_LOG_ERROR( "Data cannot be uploaded due to serialization failure" ); - return; - } - if ( mCollectionSchemeParams.compression ) - { - if ( !compress( mProtoOutput ) ) + payload.seekg( 0, std::ios::end ); + auto fileSize = static_cast( payload.tellg() ); + if ( payloadSize != fileSize ) { - FWE_LOG_ERROR( "Data cannot be uploaded due to compression failure" ); - return; + FWE_LOG_ERROR( "Failed to read persisted data: requested size " + std::to_string( payloadSize ) + + " Bytes and actual size " + std::to_string( fileSize ) + " Bytes differ for file " + + filename ); + continue; } - static_cast( send( reinterpret_cast( mCompressedProtoOutput.data() ), - mCompressedProtoOutput.size(), - mMQTTSender ) ); - } - else - { - static_cast( - send( reinterpret_cast( mProtoOutput.data() ), mProtoOutput.size(), mMQTTSender ) ); - } -} + payload.seekg( 0, std::ios::beg ); -void -DataSenderManager::checkAndSendRetrievedData() -{ - // Retrieve the metadata from persistency library - Json::Value files; - ErrorCode status = mPayloadManager->retrievePayloadMetadata( files ); - - if ( status == ErrorCode::SUCCESS ) - { - FWE_LOG_TRACE( "Number of Payloads to transmit : " + std::to_string( files.size() ) ); - for ( const auto &file : files ) - { - // Retrieve the payload data from persistency library - std::string filename = file["filename"].asString(); - - CollectionSchemeParams collectionSchemeParams; - collectionSchemeParams.compression = file["compressionRequired"].asBool(); - collectionSchemeParams.persist = true; - - size_t payloadSize = - sizeof( size_t ) >= sizeof( uint64_t ) ? file["payloadSize"].asUInt64() : file["payloadSize"].asUInt(); - if ( uploadPersistedFile( filename, payloadSize, collectionSchemeParams ) == ConnectivityError::Success ) + sender->second->processPersistedData( payload, payloadMetadata, [this, item, filename]( bool success ) { + if ( !success ) { - FWE_LOG_TRACE( "Payload from file " + filename + " has been successfully sent to the backend" ); + FWE_LOG_ERROR( "Payload transmission for file " + filename + " failed. Saving its metadata back." ); + mPayloadManager->storeMetadata( item ); + return; } - else - { - FWE_LOG_ERROR( "Payload transmission for file " + filename + " failed" ); - } - } - FWE_LOG_INFO( "Upload of persisted payloads is finished" ); - } - else - { - FWE_LOG_ERROR( "Payload Metadata Retrieval Failed" ); - } -} - -ConnectivityError -DataSenderManager::uploadPersistedFile( const std::string &filename, - size_t size, - CollectionSchemeParams collectionSchemeParams ) -{ - auto res = mMQTTSender->sendFile( filename, size, collectionSchemeParams ); - if ( res != ConnectivityError::Success ) - { - FWE_LOG_ERROR( "offboardconnectivity error " + std::to_string( static_cast( res ) ) ); + FWE_LOG_TRACE( "Payload from file " + filename + " has been successfully sent to the backend" ); + mPayloadManager->deletePayload( filename ); + } ); } - return res; } } // namespace IoTFleetWise diff --git a/src/DataSenderManager.h b/src/DataSenderManager.h index 49b667f2..6adb8c93 100644 --- a/src/DataSenderManager.h +++ b/src/DataSenderManager.h @@ -3,24 +3,11 @@ #pragma once -#include "CANInterfaceIDTranslator.h" -#include "CollectionInspectionAPITypes.h" -#include "DataSenderProtoWriter.h" -#include "IConnectionTypes.h" +#include "DataSenderTypes.h" #include "ISender.h" #include "PayloadManager.h" -#include -#include #include -#include - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "DataSenderIonWriter.h" -#include "ICollectionScheme.h" -#include "ICollectionSchemeList.h" -#include "S3Sender.h" -#include -#endif +#include namespace Aws { @@ -30,7 +17,7 @@ namespace IoTFleetWise /** * @brief Class that implements data sender logic: data preprocessing and upload * - * This class is not multithreading safe to the caller needs to ensure that the different functions + * This class is not thread-safe so the caller needs to ensure that the different functions * are called only from one thread. This class will be instantiated and used from the Data Sender * Manager Worker thread */ @@ -38,137 +25,26 @@ class DataSenderManager { public: - DataSenderManager( std::shared_ptr mqttSender, - std::shared_ptr payloadManager, - CANInterfaceIDTranslator &canIDTranslator, - unsigned transmitThreshold -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::shared_ptr s3Sender, - std::shared_ptr ionWriter, - std::string vehicleName -#endif - ); + DataSenderManager( std::unordered_map> dataSenders, + std::shared_ptr mqttSender, + std::shared_ptr payloadManager ); virtual ~DataSenderManager() = default; /** * @brief Process collection scheme parameters and prepare data for upload */ - virtual void processCollectedData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::function reportUploadCallback -#endif - ); + virtual void processData( std::shared_ptr data ); /** * @brief Retrieve all the persisted data and hand it over to the correct sender */ virtual void checkAndSendRetrievedData(); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - virtual void onChangeCollectionSchemeList( - const std::shared_ptr &activeCollectionSchemes ); -#endif - private: + std::unordered_map> mDataSenders; std::shared_ptr mMQTTSender; std::shared_ptr mPayloadManager; - DataSenderProtoWriter mProtoWriter; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - std::shared_ptr mIonWriter; - std::shared_ptr mS3Sender; // might be nullptr - std::string mVehicleName; - std::shared_ptr mActiveCollectionSchemes; -#endif - CollectionSchemeParams mCollectionSchemeParams; - std::string mCollectionSchemeID; - - std::string mProtoOutput; - std::string mCompressedProtoOutput; - - unsigned mTransmitThreshold{ 0 }; // max number of messages that can be sent to cloud at one time - - /** - * @brief Set up collectionSchemeParams struct - * @param triggeredCollectionSchemeDataPtr collected data - */ - void setCollectionSchemeParameters( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr ); - - /** - * @brief Put collected telemetry data into protobuf in chunks. Initiates serialization, compression, and - * upload for each partition. - * @param triggeredCollectionSchemeDataPtr collected data - */ - void transformTelemetryDataToProto( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr ); - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - /** - * @brief Put vision system data into ion in chunks. Initiate serialization, compression, and - * upload for each partition. - * @param triggeredCollectionSchemeDataPtr collected data - * @param uploadedDataCallback Callback after data has been successfully uploaded - */ - void transformVisionSystemDataToIon( - const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr, - std::function uploadedDataCallback ); - - S3UploadMetadata getS3UploadMetadataForCollectionScheme( const std::string &collectionSchemeID ); -#endif - - /** - * @brief Serializes, compresses, and uploads proto output. - */ - void uploadProto(); - - template - void - appendMessageToProto( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr, T msg ) - { - mProtoWriter.append( msg ); - if ( mProtoWriter.getVehicleDataMsgCount() >= mTransmitThreshold ) - { - uploadProto(); - // Setup the next payload chunk - mProtoWriter.setupVehicleData( triggeredCollectionSchemeDataPtr, mCollectionSchemeParams.eventID ); - } - } - - /** - * @brief Serializes data - * @param output Output string - * @return True if serialization succeeds - */ - bool serialize( std::string &output ); - - /** - * @brief Compresses data - * @param input Input data string - * @return True if compression succeeds - */ - bool compress( std::string &input ); - - /** - * @brief Forwards data from buffer to the provided sender - * @param data Data to send - * @param size Buffer size - * @param sender sender to use for the upload - * @return Success if upload succeeds - */ - ConnectivityError send( const std::uint8_t *data, size_t size, std::shared_ptr sender ); - - /** - * @brief Upload file from persistency folder - * @param filename File to send - * @param size File size - * @param collectionSchemeParams object containing collectionScheme related metadata for data persistency and - * transmission - * @return Success if upload succeeds - */ - ConnectivityError uploadPersistedFile( const std::string &filename, - size_t size, - CollectionSchemeParams collectionSchemeParams ); }; } // namespace IoTFleetWise diff --git a/src/DataSenderManagerWorkerThread.cpp b/src/DataSenderManagerWorkerThread.cpp index 26c6765d..47f39693 100644 --- a/src/DataSenderManagerWorkerThread.cpp +++ b/src/DataSenderManagerWorkerThread.cpp @@ -2,14 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 #include "DataSenderManagerWorkerThread.h" +#include "DataSenderTypes.h" #include "LoggingModule.h" -#include "OBDDataTypes.h" -#include "SignalTypes.h" -#include "TraceModule.h" +#include "QueueTypes.h" #include +#include #include #include -#include namespace Aws { @@ -22,8 +21,8 @@ DataSenderManagerWorkerThread::DataSenderManagerWorkerThread( std::shared_ptr connectivityModule, std::shared_ptr dataSenderManager, uint64_t persistencyUploadRetryIntervalMs, - std::shared_ptr &collectedDataQueue ) - : mCollectedDataQueue( collectedDataQueue ) + std::vector> dataToSendQueues ) + : mDataToSendQueues( std::move( dataToSendQueues ) ) , mPersistencyUploadRetryIntervalMs{ persistencyUploadRetryIntervalMs } , mDataSenderManager( std::move( dataSenderManager ) ) , mConnectivityModule( std::move( connectivityModule ) ) @@ -85,16 +84,6 @@ DataSenderManagerWorkerThread::doWork( void *data ) sender->mTimer.reset(); uint64_t minTimeToWaitMs = UINT64_MAX; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - // Check if new collection scheme is available to update available senders - if ( sender->mUpdatedCollectionSchemeListAvailable ) - { - std::lock_guard lock( sender->mActiveCollectionSchemesMutex ); - sender->mDataSenderManager->onChangeCollectionSchemeList( sender->mActiveCollectionSchemes ); - sender->mUpdatedCollectionSchemeListAvailable = false; - } -#endif - if ( sender->mPersistencyUploadRetryIntervalMs > 0 ) { uint64_t timeToWaitMs = @@ -120,119 +109,14 @@ DataSenderManagerWorkerThread::doWork( void *data ) " ms" ); } - // Dequeues the collected data queue and sends the data to cloud - auto consumeData = [&]( const TriggeredCollectionSchemeDataPtr &triggeredCollectionSchemeDataPtr ) { - // Only used for trace logging - std::string firstSignalValues = "["; - uint32_t signalPrintCounter = 0; - std::string firstSignalTimestamp; - for ( auto &s : triggeredCollectionSchemeDataPtr->signals ) - { - if ( firstSignalTimestamp.empty() ) - { - firstSignalTimestamp = " first signal timestamp: " + std::to_string( s.receiveTime ); - } - signalPrintCounter++; - if ( signalPrintCounter > MAX_NUMBER_OF_SIGNAL_TO_TRACE_LOG ) - { - firstSignalValues += " ..."; - break; - } - auto signalValue = s.getValue(); - firstSignalValues += std::to_string( s.signalID ) + ":"; - switch ( signalValue.getType() ) - { - case SignalType::UINT8: - firstSignalValues += std::to_string( signalValue.value.uint8Val ) + ","; - break; - case SignalType::INT8: - firstSignalValues += std::to_string( signalValue.value.int8Val ) + ","; - break; - case SignalType::UINT16: - firstSignalValues += std::to_string( signalValue.value.uint16Val ) + ","; - break; - case SignalType::INT16: - firstSignalValues += std::to_string( signalValue.value.int16Val ) + ","; - break; - case SignalType::UINT32: - firstSignalValues += std::to_string( signalValue.value.uint32Val ) + ","; - break; - case SignalType::INT32: - firstSignalValues += std::to_string( signalValue.value.int32Val ) + ","; - break; - case SignalType::UINT64: - firstSignalValues += std::to_string( signalValue.value.uint64Val ) + ","; - break; - case SignalType::INT64: - firstSignalValues += std::to_string( signalValue.value.int64Val ) + ","; - break; - case SignalType::FLOAT: - firstSignalValues += std::to_string( signalValue.value.floatVal ) + ","; - break; - case SignalType::DOUBLE: - firstSignalValues += std::to_string( signalValue.value.doubleVal ) + ","; - break; - case SignalType::BOOLEAN: - firstSignalValues += std::to_string( static_cast( signalValue.value.boolVal ) ) + ","; - break; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: - firstSignalValues += std::to_string( signalValue.value.uint32Val ) + ","; - break; -#endif - } - } - firstSignalValues += "]"; - // Avoid invoking Data Collection Sender if there is nothing to send. - if ( triggeredCollectionSchemeDataPtr->signals.empty() && - triggeredCollectionSchemeDataPtr->canFrames.empty() && - triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes.empty() -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - && triggeredCollectionSchemeDataPtr->uploadedS3Objects.empty() -#endif - ) - { - FWE_LOG_INFO( - "The trigger for Campaign: " + triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID + - " activated eventID: " + std::to_string( triggeredCollectionSchemeDataPtr->eventID ) + - " but no data is available to ingest" ); - } - else - { - std::string message = - "FWE data ready to send with eventID " + - std::to_string( triggeredCollectionSchemeDataPtr->eventID ) + " from " + - triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID + - " Signals:" + std::to_string( triggeredCollectionSchemeDataPtr->signals.size() ) + " " + - firstSignalValues + firstSignalTimestamp + - " trigger timestamp: " + std::to_string( triggeredCollectionSchemeDataPtr->triggerTime ) + - " raw CAN frames:" + std::to_string( triggeredCollectionSchemeDataPtr->canFrames.size() ) + - " DTCs:" + std::to_string( triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes.size() ) -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - + " Uploaded S3 Objects: " + - std::to_string( triggeredCollectionSchemeDataPtr->uploadedS3Objects.size() ) -#endif - ; - FWE_LOG_INFO( message ); - sender->mDataSenderManager->processCollectedData( - triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - [sender]( TriggeredCollectionSchemeDataPtr uploadedData ) { - if ( !sender->mCollectedDataQueue->push( std::move( uploadedData ) ) ) - { - FWE_LOG_WARN( "Collected data output buffer is full" ); - return; - } - sender->mWait.notify(); - } -#endif - ); - } - }; + size_t consumedElements = 0; + for ( auto &queue : sender->mDataToSendQueues ) + { + consumedElements += queue->consumeAll( [sender]( std::shared_ptr dataToSend ) { + sender->mDataSenderManager->processData( dataToSend ); + } ); + } - auto consumedElements = sender->mCollectedDataQueue->consumeAll( consumeData ); - TraceModule::get().setVariable( TraceVariable::QUEUE_INSPECTION_TO_SENDER, consumedElements ); if ( ( !uploadedPersistedDataOnce ) || ( ( sender->mPersistencyUploadRetryIntervalMs > 0 ) && ( static_cast( sender->mRetrySendingPersistedDataTimer.getElapsedMs().count() ) >= @@ -248,19 +132,6 @@ DataSenderManagerWorkerThread::doWork( void *data ) } } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -void -DataSenderManagerWorkerThread::onChangeCollectionSchemeList( - const std::shared_ptr &activeCollectionSchemes ) -{ - std::lock_guard lock( mActiveCollectionSchemesMutex ); - mActiveCollectionSchemes = activeCollectionSchemes; - mUpdatedCollectionSchemeListAvailable = true; - FWE_LOG_TRACE( "New list of active collections schemes was handed over" ); - mWait.notify(); -} -#endif - bool DataSenderManagerWorkerThread::isAlive() { diff --git a/src/DataSenderManagerWorkerThread.h b/src/DataSenderManagerWorkerThread.h index 951ed68b..b592d93b 100644 --- a/src/DataSenderManagerWorkerThread.h +++ b/src/DataSenderManagerWorkerThread.h @@ -3,8 +3,8 @@ #pragma once -#include "CollectionInspectionAPITypes.h" #include "DataSenderManager.h" +#include "DataSenderTypes.h" #include "IConnectivityModule.h" #include "Signal.h" #include "Thread.h" @@ -13,10 +13,7 @@ #include #include #include - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "ICollectionSchemeList.h" -#endif +#include namespace Aws { @@ -29,7 +26,7 @@ class DataSenderManagerWorkerThread DataSenderManagerWorkerThread( std::shared_ptr connectivityModule, std::shared_ptr dataSenderManager, uint64_t persistencyUploadRetryIntervalMs, - std::shared_ptr &collectedDataQueue ); + std::vector> dataToSendQueues ); ~DataSenderManagerWorkerThread(); DataSenderManagerWorkerThread( const DataSenderManagerWorkerThread & ) = delete; @@ -45,10 +42,6 @@ class DataSenderManagerWorkerThread */ void onDataReadyToPublish(); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - void onChangeCollectionSchemeList( const std::shared_ptr &activeCollectionSchemes ); -#endif - /** * @brief Stops the internal thread if started and wait until it finishes * @@ -74,7 +67,7 @@ class DataSenderManagerWorkerThread static void doWork( void *data ); - std::shared_ptr mCollectedDataQueue; + std::vector> mDataToSendQueues; uint64_t mPersistencyUploadRetryIntervalMs{ 0 }; Thread mThread; @@ -84,12 +77,6 @@ class DataSenderManagerWorkerThread std::shared_ptr mDataSenderManager; std::shared_ptr mConnectivityModule; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - std::shared_ptr mActiveCollectionSchemes; - std::mutex mActiveCollectionSchemesMutex; - std::atomic mUpdatedCollectionSchemeListAvailable{ false }; -#endif - Timer mTimer; Timer mRetrySendingPersistedDataTimer; }; diff --git a/src/DataSenderProtoWriter.cpp b/src/DataSenderProtoWriter.cpp index c41ac2b8..5b76323c 100644 --- a/src/DataSenderProtoWriter.cpp +++ b/src/DataSenderProtoWriter.cpp @@ -6,15 +6,22 @@ #include #include #include +#include + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +#include "LoggingModule.h" +#endif namespace Aws { namespace IoTFleetWise { -DataSenderProtoWriter::DataSenderProtoWriter( CANInterfaceIDTranslator &canIDTranslator ) +DataSenderProtoWriter::DataSenderProtoWriter( CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr rawDataBufferManager ) : mTriggerTime( 0U ) , mIDTranslator( canIDTranslator ) + , mRawDataBufferManager( std::move( rawDataBufferManager ) ) { } @@ -24,97 +31,134 @@ DataSenderProtoWriter::~DataSenderProtoWriter() } void -DataSenderProtoWriter::setupVehicleData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData, - uint32_t collectionEventID ) +DataSenderProtoWriter::setupVehicleData( + std::shared_ptr triggeredCollectionSchemeData, uint32_t collectionEventID ) { - mVehicleDataMsgCount = 0U; - mVehicleData.Clear(); mVehicleData.set_campaign_sync_id( triggeredCollectionSchemeData->metadata.collectionSchemeID ); mVehicleData.set_decoder_sync_id( triggeredCollectionSchemeData->metadata.decoderID ); mVehicleData.set_collection_event_id( collectionEventID ); mTriggerTime = triggeredCollectionSchemeData->triggerTime; mVehicleData.set_collection_event_time_ms_epoch( mTriggerTime ); + mMetaDataEstimatedSize = sizeof( collectionEventID ) + sizeof( mTriggerTime ) + STRING_OVERHEAD + + triggeredCollectionSchemeData->metadata.collectionSchemeID.size() + STRING_OVERHEAD + + triggeredCollectionSchemeData->metadata.decoderID.size(); + mVehicleDataEstimatedSize = mMetaDataEstimatedSize; } + void DataSenderProtoWriter::append( const CollectedSignal &msg ) { - auto capturedSignals = mVehicleData.add_captured_signals(); - mVehicleDataMsgCount++; - capturedSignals->set_relative_time_ms( static_cast( msg.receiveTime ) - - static_cast( mTriggerTime ) ); - capturedSignals->set_signal_id( msg.signalID ); + Schemas::VehicleDataMsg::CapturedSignal capturedSignal; + auto relativeTime = static_cast( msg.receiveTime ) - static_cast( mTriggerTime ); + capturedSignal.set_relative_time_ms( relativeTime ); + capturedSignal.set_signal_id( msg.signalID ); auto signalValue = msg.getValue(); // TODO :: Change the datatype of the signal here when the DataPlane supports it double signalPhysicalValue{ 0 }; + size_t size{ sizeof( msg.signalID ) + sizeof( relativeTime ) }; switch ( signalValue.getType() ) { case SignalType::UINT8: signalPhysicalValue = static_cast( signalValue.value.uint8Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::INT8: signalPhysicalValue = static_cast( signalValue.value.int8Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::UINT16: signalPhysicalValue = static_cast( signalValue.value.uint16Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::INT16: signalPhysicalValue = static_cast( signalValue.value.int16Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::UINT32: signalPhysicalValue = static_cast( signalValue.value.uint32Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::INT32: signalPhysicalValue = static_cast( signalValue.value.int32Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::UINT64: signalPhysicalValue = static_cast( signalValue.value.uint64Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::INT64: signalPhysicalValue = static_cast( signalValue.value.int64Val ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::FLOAT: signalPhysicalValue = static_cast( signalValue.value.floatVal ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::DOUBLE: signalPhysicalValue = signalValue.value.doubleVal; + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; case SignalType::BOOLEAN: signalPhysicalValue = static_cast( signalValue.value.boolVal ); + capturedSignal.set_double_value( signalPhysicalValue ); + size += sizeof( double ); break; - default: - signalPhysicalValue = signalValue.value.doubleVal; - break; + case SignalType::UNKNOWN: + // UNKNOWN signal should not be processed + return; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + FWE_LOG_WARN( "Vision System Data is not supported in the protobuf upload" ); + return; +#endif } - capturedSignals->set_double_value( signalPhysicalValue ); + auto capturedSignals = mVehicleData.add_captured_signals(); + *capturedSignals = std::move( capturedSignal ); + mVehicleDataEstimatedSize += size; } void DataSenderProtoWriter::append( const CollectedCanRawFrame &msg ) { auto rawCanFrames = mVehicleData.add_can_frames(); - mVehicleDataMsgCount++; - rawCanFrames->set_relative_time_ms( static_cast( msg.receiveTime ) - - static_cast( mTriggerTime ) ); + auto relativeTime = static_cast( msg.receiveTime ) - static_cast( mTriggerTime ); + rawCanFrames->set_relative_time_ms( relativeTime ); rawCanFrames->set_message_id( msg.frameID ); - rawCanFrames->set_interface_id( mIDTranslator.getInterfaceID( msg.channelId ) ); + auto interfaceId = mIDTranslator.getInterfaceID( msg.channelId ); + rawCanFrames->set_interface_id( interfaceId ); rawCanFrames->set_byte_values( reinterpret_cast( msg.data.data() ), msg.size ); + mVehicleDataEstimatedSize += sizeof( relativeTime ) + sizeof( msg.frameID ) + STRING_OVERHEAD + interfaceId.size() + + STRING_OVERHEAD + msg.size; } void DataSenderProtoWriter::setupDTCInfo( const DTCInfo &msg ) { auto dtcData = mVehicleData.mutable_dtc_data(); - dtcData->set_relative_time_ms( static_cast( msg.receiveTime ) - static_cast( mTriggerTime ) ); + auto relativeTime = static_cast( msg.receiveTime ) - static_cast( mTriggerTime ); + dtcData->set_relative_time_ms( relativeTime ); + mMetaDataEstimatedSize += sizeof( relativeTime ); + mVehicleDataEstimatedSize += sizeof( relativeTime ); } void DataSenderProtoWriter::append( const std::string &dtc ) { auto dtcData = mVehicleData.mutable_dtc_data(); - mVehicleDataMsgCount++; dtcData->add_active_dtc_codes( dtc ); + mVehicleDataEstimatedSize += STRING_OVERHEAD + dtc.size(); } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -122,17 +166,23 @@ void DataSenderProtoWriter::append( const UploadedS3Object &uploadedS3Object ) { auto uploadedS3Objects = mVehicleData.add_s3_objects(); - mVehicleDataMsgCount++; uploadedS3Objects->set_key( uploadedS3Object.key ); uploadedS3Objects->set_data_format( static_cast( uploadedS3Object.dataFormat ) ); + mVehicleDataEstimatedSize += sizeof( uploadedS3Object.dataFormat ) + STRING_OVERHEAD + uploadedS3Object.key.size(); } #endif -unsigned -DataSenderProtoWriter::getVehicleDataMsgCount() const +size_t +DataSenderProtoWriter::getVehicleDataEstimatedSize() const { - return mVehicleDataMsgCount; + return mVehicleDataEstimatedSize; +} + +bool +DataSenderProtoWriter::isVehicleDataAdded() const +{ + return mVehicleDataEstimatedSize > mMetaDataEstimatedSize; } bool @@ -141,5 +191,57 @@ DataSenderProtoWriter::serializeVehicleData( std::string *out ) const return mVehicleData.SerializeToString( out ); } +void +DataSenderProtoWriter::splitVehicleData( Schemas::VehicleDataMsg::VehicleData &data ) +{ + while ( mVehicleData.captured_signals_size() > data.captured_signals_size() ) + { + data.mutable_captured_signals()->AddAllocated( mVehicleData.mutable_captured_signals()->ReleaseLast() ); + } + while ( mVehicleData.can_frames_size() > data.can_frames_size() ) + { + data.mutable_can_frames()->AddAllocated( mVehicleData.mutable_can_frames()->ReleaseLast() ); + } + while ( mVehicleData.dtc_data().active_dtc_codes_size() > data.dtc_data().active_dtc_codes_size() ) + { + data.mutable_dtc_data()->mutable_active_dtc_codes()->AddAllocated( + mVehicleData.mutable_dtc_data()->mutable_active_dtc_codes()->ReleaseLast() ); + } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + while ( mVehicleData.s3_objects_size() > data.s3_objects_size() ) + { + data.mutable_s3_objects()->AddAllocated( mVehicleData.mutable_s3_objects()->ReleaseLast() ); + } +#endif +} + +void +DataSenderProtoWriter::mergeVehicleData( Schemas::VehicleDataMsg::VehicleData &data ) +{ + mVehicleData.mutable_captured_signals()->Clear(); + while ( data.captured_signals_size() > 0 ) + { + mVehicleData.mutable_captured_signals()->AddAllocated( data.mutable_captured_signals()->ReleaseLast() ); + } + mVehicleData.mutable_can_frames()->Clear(); + while ( data.can_frames_size() > 0 ) + { + mVehicleData.mutable_can_frames()->AddAllocated( data.mutable_can_frames()->ReleaseLast() ); + } + mVehicleData.mutable_dtc_data()->mutable_active_dtc_codes()->Clear(); + while ( data.dtc_data().active_dtc_codes_size() > 0 ) + { + mVehicleData.mutable_dtc_data()->mutable_active_dtc_codes()->AddAllocated( + data.mutable_dtc_data()->mutable_active_dtc_codes()->ReleaseLast() ); + } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + mVehicleData.mutable_s3_objects()->Clear(); + while ( data.s3_objects_size() > 0 ) + { + mVehicleData.mutable_s3_objects()->AddAllocated( data.mutable_s3_objects()->ReleaseLast() ); + } +#endif +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/src/DataSenderProtoWriter.h b/src/DataSenderProtoWriter.h index 853fabd2..c5422354 100644 --- a/src/DataSenderProtoWriter.h +++ b/src/DataSenderProtoWriter.h @@ -6,9 +6,12 @@ #include "CANInterfaceIDTranslator.h" #include "CollectionInspectionAPITypes.h" #include "OBDDataTypes.h" +#include "RawDataManager.h" #include "TimeTypes.h" #include "vehicle_data.pb.h" +#include #include +#include #include namespace Aws @@ -26,7 +29,8 @@ class DataSenderProtoWriter /** * @brief Constructor. Setup the DataSenderProtoWriter. */ - DataSenderProtoWriter( CANInterfaceIDTranslator &canIDTranslator ); + DataSenderProtoWriter( CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr rawDataBufferManager ); /** * @brief Destructor. @@ -45,7 +49,7 @@ class DataSenderProtoWriter * to be sent to cloud * @param collectionEventID a unique ID to tie multiple signals to a single collection event */ - void setupVehicleData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData, + void setupVehicleData( std::shared_ptr triggeredCollectionSchemeData, uint32_t collectionEventID ); /** @@ -69,6 +73,10 @@ class DataSenderProtoWriter */ void append( const std::string &dtc ); + //================================================================================================================== + // NOTE: If you add new `append` functions for new types, also add them to `splitVehicleData` and `mergeVehicleData` + //================================================================================================================== + #ifdef FWE_FEATURE_VISION_SYSTEM_DATA /** * @brief Appends uploaded S3 object info to the output protobuf @@ -86,11 +94,16 @@ class DataSenderProtoWriter void setupDTCInfo( const DTCInfo &msg ); /** - * @brief Gets the total number of collectionScheme messages sent to the cloud + * @brief Gets the estimated vehicle data size in bytes * - * @return the total number of collectionScheme messages added to the edge to cloud payload + * @return the size in bytes */ - unsigned getVehicleDataMsgCount() const; + size_t getVehicleDataEstimatedSize() const; + + /** + * @return True if any non-metadata is present in mVehicleData + */ + bool isVehicleDataAdded() const; /** * @brief Serializes the vehicle data to be sent to cloud @@ -101,11 +114,30 @@ class DataSenderProtoWriter bool serializeVehicleData( std::string *out ) const; + /** + * @brief Used when the serialized payload has exceeded the maximum payload size. + * Splits the data out to a temporary instance. + * @param data Output data + */ + void splitVehicleData( Schemas::VehicleDataMsg::VehicleData &data ); + + /** + * @brief Used when the serialized payload has exceeded the maximum payload size. + * Merges back the data from a temporary instance. + * @param data Input data + */ + void mergeVehicleData( Schemas::VehicleDataMsg::VehicleData &data ); + private: + // 2-byte overhead for LEN field, assuming strings will mostly be up to 127 bytes long. If they're larger, the size + // of the string will anyway dominate the estimated size. + static constexpr unsigned STRING_OVERHEAD = 2; Timestamp mTriggerTime; - unsigned mVehicleDataMsgCount{}; // tracks the number of messages being sent in the edge to cloud payload + size_t mMetaDataEstimatedSize{}; // The estimated size in bytes of the metadata + size_t mVehicleDataEstimatedSize{}; // The total estimated size in bytes of the mVehicleData including the metadata Schemas::VehicleDataMsg::VehicleData mVehicleData{}; CANInterfaceIDTranslator mIDTranslator; + std::shared_ptr mRawDataBufferManager; }; } // namespace IoTFleetWise diff --git a/src/DataSenderTypes.h b/src/DataSenderTypes.h new file mode 100644 index 00000000..2779cdc7 --- /dev/null +++ b/src/DataSenderTypes.h @@ -0,0 +1,185 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "QueueTypes.h" +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +/** + * @brief Defines all available types of data that can be sent + * + * Usually each type will map to a specific implementation of DataSender, which knowns how to handle + * and send that specific type. + */ +enum class SenderDataType +{ + TELEMETRY = 0, +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + VISION_SYSTEM = 1, +#endif +}; + +/** + * @brief Represents a piece of data that can be sent + * + * The concrete type can contain any kind of data, but DataSenderManager will only + * know about this abstract type and route it to the corresponding DataSender, which is expected to + * cast it to the expected concrete type. + */ +struct DataToSend +{ + virtual ~DataToSend() = default; + + virtual SenderDataType getDataType() const = 0; +}; + +/** + * @brief Represents a piece of data that should be persisted (most likely because sending it failed) + */ +struct DataToPersist +{ + virtual ~DataToPersist() = default; + + virtual SenderDataType getDataType() const = 0; + + /** + * @brief Provide the metadata to associated with this data when persisted + * + * @return A Json::Value object containing any amount of properties + */ + virtual Json::Value getMetadata() const = 0; + + /** + * @brief Provide the filename that to associated with this data when persisted + * + * @return The relative filename which will contain the data. It should contain enough details to make it unique. + */ + virtual std::string getFilename() const = 0; + + /** + * @brief Provide the underlying payload data + * + * The return type is a variant to allow implementations to choose the most appropriate type, more specifically to + * allow cases where the payload is fully in memory and cases where the payload is generated on demand. + * In any case DataSenderManager will handle any of the types and do whatever is necessary to persist the data. + * + * @return The payload data in one of the variant types. + */ + virtual boost::variant, std::shared_ptr> getData() const = 0; +}; + +/** + * @brief Map the SenderDataType enum to a string + * + * This is mostly to be used when serializing the data (e.g. in a JSON file). + * The result can be mapped back to the enum with stringToSenderDataType(). + * + * @param dataType The SenderDataType to map + * @return The string representation of the SenderDataType + */ +inline std::string +senderDataTypeToString( SenderDataType dataType ) +{ + switch ( dataType ) + { + case SenderDataType::TELEMETRY: + return "Telemetry"; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SenderDataType::VISION_SYSTEM: + return "VisionSystem"; +#endif + default: + return ""; + } +} + +/** + * @brief Map a string to a SenderDataType enum + * + * This is mostly to be used when deserializing the data (e.g. from a JSON file). + * The result can be mapped back to the string with senderDataTypeToString(). + * + * @param dataType The string representation of the SenderDataType + * @param output The reference that will contain the result if successful. + * @return Whether the conversion succeeded. If false, output will be unmodified. + */ +inline bool +stringToSenderDataType( const std::string &dataType, SenderDataType &output ) +{ + if ( dataType == "Telemetry" ) + { + output = SenderDataType::TELEMETRY; + return true; + } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + else if ( dataType == "VisionSystem" ) + { + output = SenderDataType::VISION_SYSTEM; + return true; + } +#endif + + return false; +} + +/** + * @brief Callback to be called when a data sender finished processing the data + * + * @param success true if the data was successfully processed, false otherwise + * @param data in case of failure this represents the data that needs to be persisted. It can be nullptr, which means + * that nothing should be persisted. + */ +using OnDataProcessedCallback = std::function data )>; + +/** + * @brief Callback to be called when a data sender finished processing a data that has been persisted + * + * @param success true if the data was successfully processed, false otherwise + */ +using OnPersistedDataProcessedCallback = std::function; + +/** + * @brief A sender interface to be used by DataSenderManager + * + * Each implementation knowns how to handle a specific type (or multiple) and is expected to cast the + * data to what is expected. + */ +class DataSender +{ +public: + virtual ~DataSender() = default; + + /** + * @brief Process a single piece of data. + * + * @param data The abstract data to be sent. + * @param callback The callback that will always be called to inform a success or failure. + */ + virtual void processData( std::shared_ptr data, OnDataProcessedCallback callback ) = 0; + + /** + * @brief Process a single piece of data that has been persisted before. + * + * Normally this would be already serialized data that is ready to be sent. + * + * @param data The raw data to be sent. + * @param metadata The metadata associated with the data. + * @param callback The callback that will always be called to inform a success or failure. + */ + virtual void processPersistedData( std::istream &data, + const Json::Value &metadata, + OnPersistedDataProcessedCallback callback ) = 0; +}; + +using DataSenderQueue = LockedQueue>; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/DecoderDictionaryExtractor.cpp b/src/DecoderDictionaryExtractor.cpp index aa5fecde..680dc370 100644 --- a/src/DecoderDictionaryExtractor.cpp +++ b/src/DecoderDictionaryExtractor.cpp @@ -7,6 +7,7 @@ #include "LoggingModule.h" #include "MessageTypes.h" #include +#include #include #include #include @@ -14,6 +15,8 @@ #ifdef FWE_FEATURE_VISION_SYSTEM_DATA #include +#include +#include #include #include #endif @@ -23,9 +26,202 @@ namespace Aws namespace IoTFleetWise { +void +CollectionSchemeManager::addSignalToDecoderDictionaryMap( + SignalID signalId, + std::map> &decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + std::unordered_map &partialSignalTypes, + SignalID topLevelSignalId, + SignalPath signalPath +#endif +) +{ + // get the Network Protocol Type: CAN, OBD, SOMEIP, etc + auto networkType = mDecoderManifest->getNetworkProtocol( signalId ); + if ( networkType == VehicleDataSourceProtocol::INVALID_PROTOCOL ) + { + FWE_LOG_WARN( "Invalid protocol provided for signal : " + std::to_string( signalId ) ); + // This signal contains invalid network protocol, cannot include it onto decoder dictionary + return; + } + // Firstly we need to check if we already have dictionary created for this network + if ( decoderDictionaryMap[networkType] == nullptr ) + { + if ( ( networkType == VehicleDataSourceProtocol::RAW_SOCKET ) || + ( networkType == VehicleDataSourceProtocol::OBD ) ) + { + decoderDictionaryMap[networkType] = std::make_shared(); + } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // Currently we don't have decoder dictionary for this type of network protocol, create one + else if ( networkType == VehicleDataSourceProtocol::COMPLEX_DATA ) + { + decoderDictionaryMap[networkType] = std::make_shared(); + } +#endif + else + { + FWE_LOG_ERROR( "Unknown network type: " + std::to_string( toUType( networkType ) ) + + " for signalID: " + std::to_string( signalId ) ); + return; + } + } + + if ( networkType == VehicleDataSourceProtocol::RAW_SOCKET ) + { + auto canRawFrameID = mDecoderManifest->getCANFrameAndInterfaceID( signalId ).first; + auto interfaceId = mDecoderManifest->getCANFrameAndInterfaceID( signalId ).second; + + auto canDecoderDictionaryPtr = + std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); + auto canChannelID = mCANIDTranslator.getChannelNumericID( interfaceId ); + if ( canChannelID == INVALID_CAN_SOURCE_NUMERIC_ID ) + { + FWE_LOG_WARN( "Invalid Interface ID provided: " + interfaceId ); + } + else if ( !canDecoderDictionaryPtr ) + { + FWE_LOG_WARN( "Can not cast dictionary to CANDecoderDictionary for CAN Signal ID: " + + std::to_string( signalId ) ); + } + else + { + // Add signalID to the set of this decoder dictionary + canDecoderDictionaryPtr->signalIDsToCollect.insert( signalId ); + // firstly check if we have canChannelID entry at dictionary top layer + if ( canDecoderDictionaryPtr->canMessageDecoderMethod.find( canChannelID ) == + canDecoderDictionaryPtr->canMessageDecoderMethod.end() ) + { + // create an entry for canChannelID if it's not existed yet + canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID] = + std::unordered_map(); + } + // check if this CAN Frame already exits in dictionary, if so, update if its a raw can decoder + // method. + // If not, we need to create an entry for this CAN Frame which will include decoder + // format for all signals defined in decoder manifest + if ( canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID].find( canRawFrameID ) == + canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID].end() ) + { + CANMessageDecoderMethod decoderMethod; + // We set the collect Type to DECODE at this stage. In the second half of this function, we will + // examine the CAN Frames. If there's any CAN Frame to have both signal and raw bytes to be + // collected, the type will be updated to RAW_AND_DECODE + decoderMethod.collectType = CANMessageCollectType::DECODE; + decoderMethod.format = mDecoderManifest->getCANMessageFormat( canRawFrameID, interfaceId ); + canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID] = decoderMethod; + } + else if ( canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID].collectType == + CANMessageCollectType::RAW ) + { + canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID].collectType = + CANMessageCollectType::RAW_AND_DECODE; + canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID].format = + mDecoderManifest->getCANMessageFormat( canRawFrameID, interfaceId ); + } + } + } + else if ( networkType == VehicleDataSourceProtocol::OBD ) + { + auto pidDecoderFormat = mDecoderManifest->getPIDSignalDecoderFormat( signalId ); + // There's only one VehicleDataSourceProtocol::OBD Channel, this is just a place holder to maintain the + // generic dictionary structure + CANChannelNumericID canChannelID = 0; + auto obdPidCanDecoderDictionaryPtr = + std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); + if ( !obdPidCanDecoderDictionaryPtr ) + { + FWE_LOG_WARN( "Can not cast dictionary to CANDecoderDictionary for OBD Signal ID: " + + std::to_string( signalId ) ); + } + else + { + obdPidCanDecoderDictionaryPtr->signalIDsToCollect.insert( signalId ); + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.emplace( + canChannelID, std::unordered_map() ); + if ( obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.find( canChannelID ) == + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.end() ) + { + // create an entry for canChannelID if it's not existed yet + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID] = + std::unordered_map(); + } + if ( obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) + .find( pidDecoderFormat.mPID ) == + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ).end() ) + { + // There's no Dictionary Entry created for this PID yet, create one + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) + .emplace( pidDecoderFormat.mPID, CANMessageDecoderMethod() ); + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) + .at( pidDecoderFormat.mPID ) + .format.mMessageID = pidDecoderFormat.mPID; + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) + .at( pidDecoderFormat.mPID ) + .format.mSizeInBytes = static_cast( pidDecoderFormat.mPidResponseLength ); + } + // Below is the OBD Signal format represented in generic Signal Format + CANSignalFormat format; + format.mSignalID = signalId; + format.mSignalType = pidDecoderFormat.mSignalType; + format.mFirstBitPosition = + static_cast( pidDecoderFormat.mStartByte * BYTE_SIZE + pidDecoderFormat.mBitRightShift ); + format.mSizeInBits = static_cast( ( pidDecoderFormat.mByteLength - 1 ) * BYTE_SIZE + + pidDecoderFormat.mBitMaskLength ); + format.mFactor = pidDecoderFormat.mScaling; + format.mOffset = pidDecoderFormat.mOffset; + obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) + .at( pidDecoderFormat.mPID ) + .format.mSignals.emplace_back( format ); + } + } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + else if ( networkType == VehicleDataSourceProtocol::COMPLEX_DATA ) + { + auto complexDataDictionary = + std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); + if ( !complexDataDictionary ) + { + FWE_LOG_WARN( "Can not cast dictionary to ComplexDataDecoderDictionary for Signal ID: " + + std::to_string( topLevelSignalId ) ); + } + else + { + if ( signalId != INVALID_SIGNAL_ID ) + { + auto complexSignalInfo = mDecoderManifest->getComplexSignalDecoderFormat( signalId ); + if ( complexSignalInfo.mInterfaceId.empty() ) + { + FWE_LOG_WARN( "Complex signal ID has empty interfaceID: " + std::to_string( signalId ) ); + } + else + { + auto &complexSignal = + complexDataDictionary + ->complexMessageDecoderMethod[complexSignalInfo.mInterfaceId][complexSignalInfo.mMessageId]; + putComplexSignalInDictionary( complexSignal, + signalId, + topLevelSignalId, + signalPath, + complexSignalInfo.mRootTypeId, + partialSignalTypes ); + } + } + } + } +#endif +} + void CollectionSchemeManager::decoderDictionaryExtractor( - std::map> &decoderDictionaryMap ) + std::map> &decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + std::shared_ptr inspectionMatrix +#endif +) { // Initialize the dictionary map with nullptr for each protocol, so that protocols are disabled if // none of the collection schemes collect data for that protocol @@ -34,10 +230,17 @@ CollectionSchemeManager::decoderDictionaryExtractor( { decoderDictionaryMap[protocol] = nullptr; } +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::unordered_map partialSignalTypes; +#endif // Iterate through enabled collectionScheme lists to locate the signals and CAN frames to be collected for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); ++it ) { const auto &collectionSchemePtr = it->second; + if ( collectionSchemePtr->getDecoderManifestID() != mCurrentDecoderManifestID ) + { + continue; + } // first iterate through the signalID lists for ( const auto &signalInfo : collectionSchemePtr->getCollectSignals() ) { @@ -50,7 +253,7 @@ CollectionSchemeManager::decoderDictionaryExtractor( collectionSchemePtr->getPartialSignalIdToSignalPathLookupTable().find( signalId ); if ( partialSignalInfo == collectionSchemePtr->getPartialSignalIdToSignalPathLookupTable().end() ) { - FWE_LOG_WARN( "Unknown partial signal ID: " + std::to_string( signalInfo.signalID ) ); + FWE_LOG_WARN( "Unknown partial signal ID: " + std::to_string( signalId ) ); signalId = INVALID_SIGNAL_ID; } else @@ -60,178 +263,15 @@ CollectionSchemeManager::decoderDictionaryExtractor( } } #endif - // get the Network Protocol Type: CAN, OBD, SOMEIP, etc - auto networkType = mDecoderManifest->getNetworkProtocol( signalId ); - if ( networkType == VehicleDataSourceProtocol::INVALID_PROTOCOL ) - { - FWE_LOG_WARN( "Invalid protocol provided for signal : " + std::to_string( signalId ) ); - // This signal contains invalid network protocol, cannot include it onto decoder dictionary - continue; - } - // Firstly we need to check if we already have dictionary created for this network - if ( decoderDictionaryMap[networkType] == nullptr ) - { - if ( ( networkType == VehicleDataSourceProtocol::RAW_SOCKET ) || - ( networkType == VehicleDataSourceProtocol::OBD ) ) - { - decoderDictionaryMap[networkType] = std::make_shared(); - } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - // Currently we don't have decoder dictionary for this type of network protocol, create one - else if ( networkType == VehicleDataSourceProtocol::COMPLEX_DATA ) - { - decoderDictionaryMap[networkType] = std::make_shared(); - } -#endif - else - { - FWE_LOG_ERROR( "Unknown network type: " + std::to_string( toUType( networkType ) ) + - " for signalID: " + std::to_string( signalId ) ); - continue; - } - } - - if ( networkType == VehicleDataSourceProtocol::RAW_SOCKET ) - { - auto canRawFrameID = mDecoderManifest->getCANFrameAndInterfaceID( signalId ).first; - auto interfaceId = mDecoderManifest->getCANFrameAndInterfaceID( signalId ).second; - - auto canDecoderDictionaryPtr = - std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); - auto canChannelID = mCANIDTranslator.getChannelNumericID( interfaceId ); - if ( canChannelID == INVALID_CAN_SOURCE_NUMERIC_ID ) - { - FWE_LOG_WARN( "Invalid Interface ID provided: " + interfaceId ); - } - else if ( !canDecoderDictionaryPtr ) - { - FWE_LOG_WARN( "Can not cast dictionary to CANDecoderDictionary for CAN Signal ID: " + - std::to_string( signalId ) ); - } - else - { - // Add signalID to the set of this decoder dictionary - canDecoderDictionaryPtr->signalIDsToCollect.insert( signalId ); - // firstly check if we have canChannelID entry at dictionary top layer - if ( canDecoderDictionaryPtr->canMessageDecoderMethod.find( canChannelID ) == - canDecoderDictionaryPtr->canMessageDecoderMethod.end() ) - { - // create an entry for canChannelID if it's not existed yet - canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID] = - std::unordered_map(); - } - // check if this CAN Frame already exits in dictionary, if so, update if its a raw can decoder - // method. - // If not, we need to create an entry for this CAN Frame which will include decoder - // format for all signals defined in decoder manifest - if ( canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID].find( canRawFrameID ) == - canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID].end() ) - { - CANMessageDecoderMethod decoderMethod; - // We set the collect Type to DECODE at this stage. In the second half of this function, we will - // examine the CAN Frames. If there's any CAN Frame to have both signal and raw bytes to be - // collected, the type will be updated to RAW_AND_DECODE - decoderMethod.collectType = CANMessageCollectType::DECODE; - decoderMethod.format = mDecoderManifest->getCANMessageFormat( canRawFrameID, interfaceId ); - canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID] = decoderMethod; - } - else if ( canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID] - .collectType == CANMessageCollectType::RAW ) - { - canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID].collectType = - CANMessageCollectType::RAW_AND_DECODE; - canDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID][canRawFrameID].format = - mDecoderManifest->getCANMessageFormat( canRawFrameID, interfaceId ); - } - } - } - else if ( networkType == VehicleDataSourceProtocol::OBD ) - { - auto pidDecoderFormat = mDecoderManifest->getPIDSignalDecoderFormat( signalId ); - // There's only one VehicleDataSourceProtocol::OBD Channel, this is just a place holder to maintain the - // generic dictionary structure - CANChannelNumericID canChannelID = 0; - auto obdPidCanDecoderDictionaryPtr = - std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); - if ( !obdPidCanDecoderDictionaryPtr ) - { - FWE_LOG_WARN( "Can not cast dictionary to CANDecoderDictionary for OBD Signal ID: " + - std::to_string( signalId ) ); - } - else - { - obdPidCanDecoderDictionaryPtr->signalIDsToCollect.insert( signalId ); - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.emplace( - canChannelID, std::unordered_map() ); - if ( obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.find( canChannelID ) == - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.end() ) - { - // create an entry for canChannelID if it's not existed yet - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod[canChannelID] = - std::unordered_map(); - } - if ( obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) - .find( pidDecoderFormat.mPID ) == - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ).end() ) - { - // There's no Dictionary Entry created for this PID yet, create one - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) - .emplace( pidDecoderFormat.mPID, CANMessageDecoderMethod() ); - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) - .at( pidDecoderFormat.mPID ) - .format.mMessageID = pidDecoderFormat.mPID; - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) - .at( pidDecoderFormat.mPID ) - .format.mSizeInBytes = static_cast( pidDecoderFormat.mPidResponseLength ); - } - // Below is the OBD Signal format represented in generic Signal Format - CANSignalFormat format; - format.mSignalID = signalInfo.signalID; - format.mFirstBitPosition = static_cast( pidDecoderFormat.mStartByte * BYTE_SIZE + - pidDecoderFormat.mBitRightShift ); - format.mSizeInBits = static_cast( ( pidDecoderFormat.mByteLength - 1 ) * BYTE_SIZE + - pidDecoderFormat.mBitMaskLength ); - format.mFactor = pidDecoderFormat.mScaling; - format.mOffset = pidDecoderFormat.mOffset; - obdPidCanDecoderDictionaryPtr->canMessageDecoderMethod.at( canChannelID ) - .at( pidDecoderFormat.mPID ) - .format.mSignals.emplace_back( format ); - } - } + addSignalToDecoderDictionaryMap( signalId, + decoderDictionaryMap #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - else if ( networkType == VehicleDataSourceProtocol::COMPLEX_DATA ) - { - auto complexDataDictionary = - std::dynamic_pointer_cast( decoderDictionaryMap[networkType] ); - if ( !complexDataDictionary ) - { - FWE_LOG_WARN( "Can not cast dictionary to ComplexDataDecoderDictionary for Signal ID: " + - std::to_string( signalInfo.signalID ) ); - } - else - { - if ( signalId != INVALID_SIGNAL_ID ) - { - auto complexSignalInfo = mDecoderManifest->getComplexSignalDecoderFormat( signalId ); - if ( complexSignalInfo.mInterfaceId.empty() ) - { - FWE_LOG_WARN( "Complex signal ID has empty interfaceID: " + std::to_string( signalId ) ); - } - else - { - auto &complexSignal = - complexDataDictionary->complexMessageDecoderMethod[complexSignalInfo.mInterfaceId] - [complexSignalInfo.mMessageId]; - putComplexSignalInDictionary( complexSignal, - signalId, - signalInfo.signalID, - signalPath, - complexSignalInfo.mRootTypeId ); - } - } - } - } + , + partialSignalTypes, + signalInfo.signalID, + signalPath #endif + ); } // Next let's iterate through the CAN Frames that collectionScheme wants to collect. // If some CAN Frame has signals to be decoded, we will set its collectType as RAW_AND_DECODE. @@ -292,15 +332,114 @@ CollectionSchemeManager::decoderDictionaryExtractor( } } } + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // Now we need to update the InspectionMatrix to set the correct SignalType for partial signal IDs. + // We do this here because by the time the InspectionMatrix is created we don't have enough info + // to determine the type of partial signals. + // So any signal that is a partial signal needs to be updated here, otherwise they would have the + // wrong signal type which would cause a mismatch with the type used to set up CollectionInspectionEngine + // buffers. + if ( !partialSignalTypes.empty() ) + { + for ( auto &condition : inspectionMatrix->conditions ) + { + for ( auto &signal : condition.signals ) + { + auto signalType = partialSignalTypes.find( signal.signalID ); + if ( signalType != partialSignalTypes.end() ) + { + FWE_LOG_TRACE( + "Signal type for partial signal with internal ID: " + std::to_string( signal.signalID ) + + " is being overwritten with type: " + signalTypeToString( signalType->second ) ); + signal.signalType = signalType->second; + } + } + } + } +#endif } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA +static boost::optional +findPartialSignalType( const ComplexDataMessageFormat &complexSignal, const SignalPathAndPartialSignalID &signalPath ) +{ + auto currentTypeId = complexSignal.mRootTypeId; + auto complexDataType = complexSignal.mComplexTypeMap.find( currentTypeId ); + + for ( auto &pathLevel : signalPath.mSignalPath ) + { + if ( complexDataType == complexSignal.mComplexTypeMap.end() ) + { + FWE_LOG_ERROR( "Could not find type for ID: " + std::to_string( currentTypeId ) ); + return boost::none; + } + + if ( complexDataType->second.type() == typeid( ComplexStruct ) ) + { + try + { + auto complexStruct = boost::get( complexDataType->second ); + currentTypeId = complexStruct.mOrderedTypeIds[pathLevel]; + } + catch ( boost::bad_get & ) + { + // Should not throw because of typeid check but catch it to prevent static analysis errors + } + } + else if ( complexDataType->second.type() == typeid( ComplexArray ) ) + { + try + { + auto complexArray = boost::get( complexDataType->second ); + currentTypeId = complexArray.mRepeatedTypeId; + } + catch ( boost::bad_get & ) + { + // Should not throw because of typeid check but catch it to prevent static analysis errors + } + } + else + { + break; + } + + complexDataType = complexSignal.mComplexTypeMap.find( currentTypeId ); + } + + if ( complexDataType == complexSignal.mComplexTypeMap.end() ) + { + FWE_LOG_ERROR( "Could not find partial type" ); + return boost::none; + } + + if ( complexDataType->second.type() != typeid( PrimitiveData ) ) + { + FWE_LOG_TRACE( "Signal path pointing to type ID: " + std::to_string( complexDataType->first ) + + " is not a primitive type" ); + return boost::none; + } + + try + { + auto primitiveData = boost::get( complexDataType->second ); + return primitiveData.mPrimitiveType; + } + catch ( boost::bad_get & ) + { + // Should not throw because of typeid check but catch it to prevent static analysis errors + } + + return boost::none; +} + void CollectionSchemeManager::putComplexSignalInDictionary( ComplexDataMessageFormat &complexSignal, SignalID signalID, PartialSignalID partialSignalID, SignalPath &signalPath, - ComplexDataTypeId complexSignalRootType ) + ComplexDataTypeId complexSignalRootType, + std::unordered_map &partialSignalTypes ) { if ( complexSignal.mSignalId == INVALID_SIGNAL_ID ) { @@ -331,12 +470,12 @@ CollectionSchemeManager::putComplexSignalInDictionary( ComplexDataMessageFormat { try { - auto t = boost::get( complexDataType ) - .mRepeatedTypeId; // Should not trow because of typeid check but catch anyway + auto t = boost::get( complexDataType ).mRepeatedTypeId; complexTypesToTraverse.push( t ); } - catch ( ... ) + catch ( boost::bad_get & ) { + // Should not throw because of typeid check but catch it to prevent static analysis errors } } if ( complexDataType.type() == typeid( ComplexStruct ) ) @@ -348,8 +487,9 @@ CollectionSchemeManager::putComplexSignalInDictionary( ComplexDataMessageFormat complexTypesToTraverse.push( member ); } } - catch ( ... ) + catch ( boost::bad_get & ) { + // Should not throw because of typeid check but catch it to prevent static analysis errors } } } @@ -368,6 +508,12 @@ CollectionSchemeManager::putComplexSignalInDictionary( ComplexDataMessageFormat complexSignal.mSignalPaths.insert( std::upper_bound( complexSignal.mSignalPaths.begin(), complexSignal.mSignalPaths.end(), newPathToInsert ), newPathToInsert ); + + auto signalType = findPartialSignalType( complexSignal, newPathToInsert ); + if ( signalType.has_value() ) + { + partialSignalTypes[partialSignalID] = signalType.get(); + } } } #endif diff --git a/src/DecoderManifestIngestion.cpp b/src/DecoderManifestIngestion.cpp index 02be347a..ba0a1d1c 100644 --- a/src/DecoderManifestIngestion.cpp +++ b/src/DecoderManifestIngestion.cpp @@ -6,6 +6,7 @@ #include "EnumUtility.h" #include "LoggingModule.h" #include "OBDDataTypes.h" +#include #include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -23,13 +24,13 @@ DecoderManifestIngestion::~DecoderManifestIngestion() google::protobuf::ShutdownProtobufLibrary(); } -std::string +SyncID DecoderManifestIngestion::getID() const { if ( !mReady ) { // Return empty string - return std::string(); + return SyncID(); } return mProtoDecoderManifest.sync_id(); @@ -230,6 +231,10 @@ DecoderManifestIngestion::build() mSignalToCANRawFrameIDAndInterfaceIDDictionary.insert( std::make_pair( canSignal.signal_id(), std::make_pair( canSignal.message_id(), canSignal.interface_id() ) ) ); + // For backward compatibility, default to double + auto signalType = + convertPrimitiveTypeToSignalType( canSignal.primitive_type() ).get_value_or( SignalType::DOUBLE ); + // Create a container to hold the InterfaceManagement::CANSignal we will build CANSignalFormat canSignalFormat; @@ -240,14 +245,15 @@ DecoderManifestIngestion::build() canSignalFormat.mSizeInBits = static_cast( canSignal.length() ); canSignalFormat.mOffset = canSignal.offset(); canSignalFormat.mFactor = canSignal.factor(); + canSignalFormat.mSignalType = signalType; - // TODO :: Update the datatype from the DM after the schema update for the datatype support - mSignalIDToTypeMap[canSignal.signal_id()] = SignalType::DOUBLE; // using double as default + mSignalIDToTypeMap[canSignal.signal_id()] = signalType; canSignalFormat.mIsMultiplexorSignal = false; canSignalFormat.mMultiplexorValue = 0; - FWE_LOG_TRACE( "Adding CAN Signal Format for Signal ID: " + std::to_string( canSignalFormat.mSignalID ) ); + FWE_LOG_TRACE( "Adding CAN Signal Format for Signal ID: " + std::to_string( canSignalFormat.mSignalID ) + + " and message ID: " + std::to_string( canSignal.message_id() ) ); // Each CANMessageFormat object contains an array of signal decoding rules for each signal it contains. Cloud // sends us a set of Signal IDs so we need to iterate through them and either create new CANMessageFormat @@ -307,6 +313,9 @@ DecoderManifestIngestion::build() continue; } mSignalToVehicleDataSourceProtocol[pidSignal.signal_id()] = VehicleDataSourceProtocol::OBD; + // For backward compatibility, default to double + auto signalType = + convertPrimitiveTypeToSignalType( pidSignal.primitive_type() ).get_value_or( SignalType::DOUBLE ); PIDSignalDecoderFormat obdPIDSignalDecoderFormat = PIDSignalDecoderFormat( pidSignal.pid_response_length(), // coverity[autosar_cpp14_a7_2_1_violation] The if-statement above checks the correct range @@ -318,9 +327,9 @@ DecoderManifestIngestion::build() pidSignal.byte_length(), static_cast( pidSignal.bit_right_shift() ), static_cast( pidSignal.bit_mask_length() ) ); + obdPIDSignalDecoderFormat.mSignalType = signalType; mSignalToPIDDictionary[pidSignal.signal_id()] = obdPIDSignalDecoderFormat; - // TODO :: Update the datatype from the DM after the schema update for the datatype support - mSignalIDToTypeMap[pidSignal.signal_id()] = SignalType::DOUBLE; // using double as default + mSignalIDToTypeMap[pidSignal.signal_id()] = signalType; } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -346,7 +355,8 @@ DecoderManifestIngestion::build() auto scaling = primitiveData.scaling(); scaling = ( scaling == 0.0 ? 1.0 : scaling ); // If scaling is not set in protobuf or set to invalid 0 // replace it by default scaling: 1 - auto convertedType = convertPrimitiveTypeToSignalType( primitiveData.primitive_type() ); + auto convertedType = + convertPrimitiveTypeToSignalType( primitiveData.primitive_type() ).get_value_or( SignalType::UINT8 ); mComplexTypeMap[complexType.type_id()] = ComplexDataElement( PrimitiveData{ convertedType, scaling, primitiveData.offset() } ); FWE_LOG_TRACE( "Adding PrimitiveData with complex type id: " + std::to_string( complexType.type_id() ) + @@ -375,22 +385,21 @@ DecoderManifestIngestion::build() else if ( complexType.variant_case() == Schemas::DecoderManifestMsg::ComplexType::kStringData ) { auto &complexStringData = complexType.string_data(); - if ( ( complexStringData.encoding() != Schemas::DecoderManifestMsg::StringEncoding::UTF_16 ) && - ( complexStringData.encoding() != Schemas::DecoderManifestMsg::StringEncoding::UTF_8 ) ) + auto encoding = complexStringData.encoding(); + if ( ( encoding != Schemas::DecoderManifestMsg::StringEncoding::UTF_16 ) && + ( encoding != Schemas::DecoderManifestMsg::StringEncoding::UTF_8 ) ) { FWE_LOG_WARN( "String data with type id " + std::to_string( complexType.type_id() ) + - " has invalid encoding: " + - std::to_string( static_cast( complexStringData.encoding() ) ) ); + " has invalid encoding: " + std::to_string( static_cast( encoding ) ) ); continue; } - ComplexDataTypeId characterType = - complexStringData.encoding() == Schemas::DecoderManifestMsg::StringEncoding::UTF_16 - ? RESERVED_UTF16_UINT32_TYPE_ID - : RESERVED_UTF8_UINT8_TYPE_ID; + ComplexDataTypeId characterType = encoding == Schemas::DecoderManifestMsg::StringEncoding::UTF_16 + ? RESERVED_UTF16_UINT32_TYPE_ID + : RESERVED_UTF8_UINT8_TYPE_ID; if ( mComplexTypeMap.find( characterType ) == mComplexTypeMap.end() ) { SignalType signalType = - complexStringData.encoding() == Schemas::DecoderManifestMsg::StringEncoding::UTF_16 + encoding == Schemas::DecoderManifestMsg::StringEncoding::UTF_16 ? SignalType::UINT32 // ROS2 implementation uses uint32 for utf-16 (wstring) code units : SignalType::UINT8; mComplexTypeMap[characterType] = PrimitiveData{ signalType, 1.0, 0.0 }; @@ -418,7 +427,7 @@ DecoderManifestIngestion::build() complexSignal.interface_id(), complexSignal.message_id(), complexSignal.root_type_id() }; mSignalIDToTypeMap[complexSignal.signal_id()] = - SignalType::RAW_DATA_BUFFER_HANDLE; // handle top level signals always as raw data handles + SignalType::COMPLEX_SIGNAL; // handle top level signals always as raw data handles FWE_LOG_TRACE( "Adding complex signal with id: " + std::to_string( complexSignal.signal_id() ) + " with interface ID: '" + complexSignal.interface_id() + "' message ID: '" + complexSignal.message_id() + @@ -433,8 +442,7 @@ DecoderManifestIngestion::build() return true; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -SignalType +boost::optional DecoderManifestIngestion::convertPrimitiveTypeToSignalType( Schemas::DecoderManifestMsg::PrimitiveType primitiveType ) { switch ( primitiveType ) @@ -461,13 +469,14 @@ DecoderManifestIngestion::convertPrimitiveTypeToSignalType( Schemas::DecoderMani return SignalType::FLOAT; case Schemas::DecoderManifestMsg::PrimitiveType::FLOAT64: return SignalType::DOUBLE; + case Schemas::DecoderManifestMsg::PrimitiveType::NULL_: + return SignalType::DOUBLE; default: FWE_LOG_WARN( "Currently PrimitiveType " + std::to_string( primitiveType ) + " is not supported" ); break; } - return SignalType::UINT8; // default to uint8 + return boost::none; } -#endif } // namespace IoTFleetWise } // namespace Aws diff --git a/src/DecoderManifestIngestion.h b/src/DecoderManifestIngestion.h index 2d2bc74e..2da29282 100644 --- a/src/DecoderManifestIngestion.h +++ b/src/DecoderManifestIngestion.h @@ -9,6 +9,7 @@ #include "SignalTypes.h" #include "VehicleDataSourceTypes.h" #include "decoder_manifest.pb.h" +#include #include #include #include @@ -52,7 +53,7 @@ class DecoderManifestIngestion : public IDecoderManifest bool build() override; - std::string getID() const override; + SyncID getID() const override; const CANMessageFormat &getCANMessageFormat( CANRawFrameID canID, InterfaceID interfaceID ) const override; @@ -82,8 +83,8 @@ class DecoderManifestIngestion : public IDecoderManifest if ( mSignalIDToTypeMap.find( signalID ) == mSignalIDToTypeMap.end() ) { FWE_LOG_WARN( "Signal Type not found for requested SignalID:" + std::to_string( signalID ) + - ", using type as double" ); - return SignalType::DOUBLE; + ", using type as unknown" ); + return SignalType::UNKNOWN; } return mSignalIDToTypeMap.at( signalID ); } @@ -143,6 +144,7 @@ class DecoderManifestIngestion : public IDecoderManifest * that can be looked up again in this map. */ std::unordered_map mComplexTypeMap; +#endif /** * @brief In the protobuf a different enum if used to represent the different types like uint8, uint16 etc. @@ -150,8 +152,8 @@ class DecoderManifestIngestion : public IDecoderManifest * * @return the type C++ enum used by the DecoderDictionary */ - static SignalType convertPrimitiveTypeToSignalType( Schemas::DecoderManifestMsg::PrimitiveType primitiveType ); -#endif + static boost::optional convertPrimitiveTypeToSignalType( + Schemas::DecoderManifestMsg::PrimitiveType primitiveType ); }; } // namespace IoTFleetWise diff --git a/src/ExternalGpsSource.cpp b/src/ExternalGpsSource.cpp index d6e84271..f2eb7873 100644 --- a/src/ExternalGpsSource.cpp +++ b/src/ExternalGpsSource.cpp @@ -2,8 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 #include "ExternalGpsSource.h" +#include "CollectionInspectionAPITypes.h" #include "LoggingModule.h" -#include "TraceModule.h" +#include "QueueTypes.h" +#include "SignalTypes.h" #include #include @@ -17,8 +19,8 @@ constexpr const char *ExternalGpsSource::CAN_CHANNEL_NUMBER; // NOLINT constexpr const char *ExternalGpsSource::CAN_RAW_FRAME_ID; // NOLINT constexpr const char *ExternalGpsSource::LATITUDE_START_BIT; // NOLINT constexpr const char *ExternalGpsSource::LONGITUDE_START_BIT; // NOLINT -ExternalGpsSource::ExternalGpsSource( SignalBufferPtr signalBufferPtr ) - : mSignalBufferPtr{ std::move( signalBufferPtr ) } +ExternalGpsSource::ExternalGpsSource( SignalBufferDistributorPtr signalBufferDistributor ) + : mSignalBufferDistributor{ std::move( signalBufferDistributor ) } { } @@ -42,7 +44,7 @@ ExternalGpsSource::init( CANChannelNumericID canChannel, const char * ExternalGpsSource::getThreadName() { - return "ExternalGpsSource"; + return "ExternalGpsSrc"; } void @@ -64,18 +66,10 @@ ExternalGpsSource::setLocation( double latitude, double longitude ) } auto timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( latitudeSignalId, timestamp, latitude ) ); - collectedSignalsGroup.push_back( CollectedSignal( longitudeSignalId, timestamp, longitude ) ); + collectedSignalsGroup.push_back( CollectedSignal( latitudeSignalId, timestamp, latitude, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( longitudeSignalId, timestamp, longitude, SignalType::DOUBLE ) ); - TraceModule::get().addToAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, 2 ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - - if ( !mSignalBufferPtr->push( CollectedDataFrame( std::move( collectedSignalsGroup ) ) ) ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().subtractFromAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, 2 ); - FWE_LOG_WARN( "Signal buffer full" ); - } + mSignalBufferDistributor->push( CollectedDataFrame( std::move( collectedSignalsGroup ) ) ); } void diff --git a/src/ExternalGpsSource.h b/src/ExternalGpsSource.h index f53f969b..51b2de4a 100644 --- a/src/ExternalGpsSource.h +++ b/src/ExternalGpsSource.h @@ -23,9 +23,9 @@ class ExternalGpsSource : public CustomDataSource { public: /** - * @param signalBufferPtr the signal buffer is used pass extracted data + * @param signalBufferDistributor Signal buffer distributor */ - ExternalGpsSource( SignalBufferPtr signalBufferPtr ); + ExternalGpsSource( SignalBufferDistributorPtr signalBufferDistributor ); /** * Initialize ExternalGpsSource and set filter for CustomDataSource * @@ -59,7 +59,7 @@ class ExternalGpsSource : public CustomDataSource uint16_t mLatitudeStartBit = 0; uint16_t mLongitudeStartBit = 0; - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; std::shared_ptr mClock = ClockHandler::getClock(); CANChannelNumericID mCanChannel{ INVALID_CAN_SOURCE_NUMERIC_ID }; CANRawFrameID mCanRawFrameId{ 0 }; diff --git a/src/ICollectionScheme.h b/src/ICollectionScheme.h index dff9d0e2..ef6c8a96 100644 --- a/src/ICollectionScheme.h +++ b/src/ICollectionScheme.h @@ -9,7 +9,6 @@ #include #include #include -#include #include namespace Aws @@ -81,8 +80,7 @@ struct CanFrameCollectionInfo enum class ExpressionNodeType { FLOAT = 0, - SIGNAL, // Node_Signal_ID - WINDOWFUNCTION, // NodeFunction + SIGNAL, // Node_Signal_ID BOOLEAN, OPERATOR_SMALLER, // NodeOperator OPERATOR_BIGGER, @@ -96,7 +94,9 @@ enum class ExpressionNodeType OPERATOR_ARITHMETIC_PLUS, OPERATOR_ARITHMETIC_MINUS, OPERATOR_ARITHMETIC_MULTIPLY, - OPERATOR_ARITHMETIC_DIVIDE + OPERATOR_ARITHMETIC_DIVIDE, + WINDOW_FUNCTION, // NodeFunction + NONE }; enum class WindowFunction @@ -112,6 +112,9 @@ enum class WindowFunction struct ExpressionFunction { + /** + * @brief Specify which window function to be used (in case ExpressionNodeType is WINDOW_FUNCTION). + */ WindowFunction windowFunction{ WindowFunction::NONE }; }; @@ -245,18 +248,22 @@ class ICollectionScheme #endif /** - * @brief Signals_t is a vector that represents the AST Expression Tree per collectionScheme provided. + * @brief ExpressionNode_t is a vector that represents the AST Expression Tree per collectionScheme provided. */ using ExpressionNode_t = std::vector; - const ExpressionNode_t INVALID_EXPRESSION_NODE = std::vector(); + const ExpressionNode_t INVALID_EXPRESSION_NODES = std::vector(); const uint64_t INVALID_COLLECTION_SCHEME_START_TIME = std::numeric_limits::max(); const uint64_t INVALID_COLLECTION_SCHEME_EXPIRY_TIME = std::numeric_limits::max(); const uint32_t INVALID_MINIMUM_PUBLISH_TIME = std::numeric_limits::max(); const uint32_t INVALID_AFTER_TRIGGER_DURATION = std::numeric_limits::max(); const uint32_t INVALID_PRIORITY_LEVEL = std::numeric_limits::max(); - const std::string INVALID_COLLECTION_SCHEME_ID = std::string(); - const std::string INVALID_DECODER_MANIFEST_ID = std::string(); + const SyncID INVALID_COLLECTION_SCHEME_ID = SyncID(); + const SyncID INVALID_DECODER_MANIFEST_ID = SyncID(); + + virtual bool operator==( const ICollectionScheme &other ) const = 0; + + virtual bool operator!=( const ICollectionScheme &other ) const = 0; /** * @brief indicates if the decoder manifest is prepared to be used for example by calling getters @@ -277,14 +284,14 @@ class ICollectionScheme * * @return A string containing the unique ID of this collectionScheme. */ - virtual const std::string &getCollectionSchemeID() const = 0; + virtual const SyncID &getCollectionSchemeID() const = 0; /** * @brief Get the associated Decoder Manifest ID of this collectionScheme * * @return A string containing the unique Decoder Manifest of this collectionScheme. */ - virtual const std::string &getDecoderManifestID() const = 0; + virtual const SyncID &getDecoderManifestID() const = 0; /** * @brief Get Activation timestamp of the collectionScheme diff --git a/src/ICollectionSchemeManager.h b/src/ICollectionSchemeManager.h deleted file mode 100644 index 4780d3a1..00000000 --- a/src/ICollectionSchemeManager.h +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "CacheAndPersist.h" -#include "CollectionInspectionAPITypes.h" -#include "ICollectionScheme.h" -#include "IDecoderManifest.h" - -namespace Aws -{ -namespace IoTFleetWise -{ - -class ICollectionSchemeManager -{ -public: - virtual ~ICollectionSchemeManager() = default; - - /** - * @brief callback for CollectionScheme Ingestion to send update of ICollectionSchemeList - * @param collectionSchemeList a constant shared pointer to ICollectionSchemeList from CollectionScheme Ingestion - */ - virtual void onCollectionSchemeUpdate( const ICollectionSchemeListPtr &collectionSchemeList ) = 0; - - /** - * @brief callback for CollectionScheme Ingestion to send update of IDecoderManifest - * @param decoderManifest a constant shared pointer to IDecoderManifest from CollectionScheme Ingestion - */ - virtual void onDecoderManifestUpdate( const IDecoderManifestPtr &decoderManifest ) = 0; - -protected: - /** - * @brief Rebuilds enabled collectionScheme map, idle collectionScheme map, and time line; - * This function runs after change of decoderManifest is detected. - * - * @param currTime time in seconds when main thread wakes up; - * @return true when enabled collectionScheme map is updated. - */ - virtual bool rebuildMapsandTimeLine( const TimePoint &currTime ) = 0; - - /** - * @brief Updates existing enabled collectionScheme map, idle collectionScheme map, and time line; - * This function runs when receiving an update from CollectionScheme Ingestion with no decoderManifest change. - * It detects new collectionScheme on the list and updates collectionScheme maps and time line, as well as missing - * collectionSchemes, removes them from maps and calls callback functions when needed. - * - * @param currTime time in seconds when main thread wakes up; - * @return true when enabled collectionScheme map is updated. - */ - virtual bool updateMapsandTimeLine( const TimePoint &currTime ) = 0; - - /** - * @brief works on TimeData popped from mTimeLine to decide whether to disable/enable a collectionScheme - * - * @param currTime time in seconds when main thread wakes up; - * @return true when enabled collectionScheme map is updated. - */ - virtual bool checkTimeLine( const TimePoint &currTime ) = 0; - - /** - * @brief Extract from enabled collectionSchemes and aggregate into inspectMatrix - * - * @param inspectionMatrix the inspectionMatrix object to be filled - * - */ - virtual void inspectionMatrixExtractor( const std::shared_ptr &inspectionMatrix ) = 0; - - /** - * @brief This function invoke all the listener for inspectMatrix update - * - * @param inspectionMatrix the inspectionMatrix object to be filled - * - */ - virtual void inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) = 0; - - /** - * @brief Retrieves Protobuf-ed collectionSchemeList or decoderManifest from - * persistent memory - * - * @param retrieveType ENUM DataType defined in CacheAndPersist.h - */ - virtual bool retrieve( DataType retrieveType ) = 0; - - /** - * @brief Stores protobuf-ed collectionSchemeList or DecoderManifest to - * persistent memory - * - * @param storeType ENUM DataType defined in CacheAndPersist.h - * - */ - virtual void store( DataType storeType ) = 0; - - /** - * @brief processes proto-bufed decoder manifest - * - */ - virtual bool processDecoderManifest() = 0; - - /** - * @brief processes proto-bufed collectionScheme - * - */ - virtual bool processCollectionScheme() = 0; - - /** - * @brief pack checkin message and send out - * @return True if the Connectivity Module packed and send the data out of the process space. - * It does not guarantee that the data reached the Checkin topic ( best effort QoS ) - */ - virtual bool sendCheckin() = 0; - - /** - * @brief checks if mCollectionSchemeAvailable and mDecoderManifestAvailable is set and - * copies pointers out of critical section - * - */ - virtual void updateAvailable() = 0; -}; - -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/IConnectionTypes.h b/src/IConnectionTypes.h index 3f84c870..c3e5b694 100644 --- a/src/IConnectionTypes.h +++ b/src/IConnectionTypes.h @@ -3,6 +3,8 @@ #pragma once +#include + namespace Aws { namespace IoTFleetWise @@ -13,13 +15,38 @@ namespace IoTFleetWise */ enum class ConnectivityError { - Success = 0, /**< everything OK, still no guarantee that data was transmitted correctly */ - NoConnection, /**< currently no connection, the Connectivity module will try to reestablish it automatically */ - QuotaReached, /**< quota reached for example outgoing queue full so please try again after few milliseconds */ - NotConfigured, /**< the object used was not configured correctly */ - WrongInputData, /**< invalid input data was provided */ - TypeNotSupported, /**< requested upload type is not supported by the sender */ + Success = 0, /**< everything OK, still no guarantee that data was transmitted correctly */ + NoConnection, /**< currently no connection, the Connectivity module will try to reestablish it automatically */ + QuotaReached, /**< quota reached for example outgoing queue full so please try again after few milliseconds */ + NotConfigured, /**< the object used was not configured correctly */ + WrongInputData, /**< invalid input data was provided */ + TypeNotSupported, /**< requested upload type is not supported by the sender */ + TransmissionError, /**< some error happened after start transmission of data */ }; +static inline std::string +connectivityErrorToString( ConnectivityError error ) +{ + switch ( error ) + { + case ConnectivityError::Success: + return "Success"; + case ConnectivityError::NoConnection: + return "NoConnection"; + case ConnectivityError::QuotaReached: + return "QuotaReached"; + case ConnectivityError::NotConfigured: + return "NotConfigured"; + case ConnectivityError::WrongInputData: + return "WrongInputData"; + case ConnectivityError::TypeNotSupported: + return "TypeNotSupported"; + case ConnectivityError::TransmissionError: + return "TransmissionError"; + } + + return "Undefined"; +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/src/IConnectivityChannel.h b/src/IConnectivityChannel.h deleted file mode 100644 index 3c9ea6e0..00000000 --- a/src/IConnectivityChannel.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "IReceiver.h" -#include "ISender.h" -#include "PayloadManager.h" -#include -#include -#include -#include -#include - -namespace Aws -{ -namespace IoTFleetWise -{ - -class IConnectivityChannel : public ISender, public IReceiver -{ -public: - ~IConnectivityChannel() override = default; - - virtual unsigned getPayloadCountSent() const = 0; -}; -} // namespace IoTFleetWise -} // namespace Aws diff --git a/src/IConnectivityModule.h b/src/IConnectivityModule.h index e60ce66c..8e23d21c 100644 --- a/src/IConnectivityModule.h +++ b/src/IConnectivityModule.h @@ -3,8 +3,8 @@ #pragma once -#include "IConnectivityChannel.h" -#include "PayloadManager.h" +#include "IReceiver.h" +#include "ISender.h" #include #include #include @@ -15,6 +15,17 @@ namespace Aws namespace IoTFleetWise { +enum class QoS +{ + AT_MOST_ONCE = 0, + AT_LEAST_ONCE = 1, +}; + +/** + * @brief called after the mqtt client is connected + */ +using OnConnectionEstablishedCallback = std::function; + class IConnectivityModule { public: @@ -22,14 +33,38 @@ class IConnectivityModule virtual bool isAlive() const = 0; - virtual std::shared_ptr createNewChannel( - const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription = false ) = 0; + /** + * @brief create a new sender sharing the connection of this module + * + * @param topicName the topic which this sender should publish to + * @param publishQoS the QoS level for the publish messages + * + * @return a pointer to the newly created sender. A reference to the newly created sender is also hold inside this + * module. + */ + virtual std::shared_ptr createSender( const std::string &topicName, + QoS publishQoS = QoS::AT_MOST_ONCE ) = 0; + + /** + * @brief create a new receiver sharing the connection of this module + * + * @param topicName the topic which this receiver should subscribe to + * + * @return a pointer to the newly created receiver. A reference to the newly created receiver is also hold inside + * this module. + */ + virtual std::shared_ptr createReceiver( const std::string &topicName ) = 0; virtual bool disconnect() = 0; virtual bool connect() = 0; + + /** + * @brief Subscribe to the event that is triggered when the connection is established + * + * @param callback The callback to be called when the connection is established. + */ + virtual void subscribeToConnectionEstablished( OnConnectionEstablishedCallback callback ) = 0; }; } // namespace IoTFleetWise diff --git a/src/IDecoderManifest.h b/src/IDecoderManifest.h index 4feef586..3116604a 100644 --- a/src/IDecoderManifest.h +++ b/src/IDecoderManifest.h @@ -110,9 +110,9 @@ struct PIDSignalDecoderFormat uint8_t mBitMaskLength{ 0 }; /** - * @brief The datatype of the signal. The default is double for backward compatibility + * @brief The datatype of the signal. */ - SignalType mSignalType{ SignalType::DOUBLE }; + SignalType mSignalType{ SignalType::UNKNOWN }; public: /** @@ -194,7 +194,7 @@ class IDecoderManifest * * @return String ID of the decoder manifest. Empty string if error. */ - virtual std::string getID() const = 0; + virtual SyncID getID() const = 0; /** * @brief get CAN Message format to decode. diff --git a/src/IReceiver.h b/src/IReceiver.h index 177eff9f..8eeb5d25 100644 --- a/src/IReceiver.h +++ b/src/IReceiver.h @@ -14,8 +14,19 @@ namespace Aws namespace IoTFleetWise { -struct ReceivedChannelMessage +struct ReceivedConnectivityMessage { +public: + ReceivedConnectivityMessage( const std::uint8_t *bufIn, + size_t sizeIn, + Timestamp receivedMonotonicTimeMsIn, + std::string mqttTopicIn ) + : buf( bufIn ) + , size( sizeIn ) + , receivedMonotonicTimeMs( receivedMonotonicTimeMsIn ) + , mqttTopic( std::move( mqttTopicIn ) ) + { + } /* * Pointer to raw received data that will be at least size long. * The function does not care if the data is a c string, a json or a binary @@ -29,14 +40,14 @@ struct ReceivedChannelMessage size_t size{ 0 }; /* - * Key/value pairs that were received together with the data. It might be empty. + * Time when this message was received. The monotonic clock is used (which is not necessarily a Unix timestamp) */ - const std::unordered_map &properties; + Timestamp receivedMonotonicTimeMs{ 0 }; /* - * Absolute MQTT message expiry time since epoch from a monotonic clock. + * MQTT topic name */ - Timestamp messageExpiryMonotonicTimeSinceEpochMs{ 0 }; + std::string mqttTopic; }; /** @@ -47,36 +58,19 @@ struct ReceivedChannelMessage * The function behind onDataReceived must be fast (<1ms) and the pointer buf will get * invalid after returning from the callback. * - * @param receivedChannelMessage struct containing message data and metadata + * @param receivedMessage struct containing message data and metadata */ -using OnDataReceivedCallback = std::function; - -// Define some common property names to make it easier for subscribers to extract the properties they -// are interested in when the callback is called. -constexpr auto PROPERTY_NAME_CORRELATION_DATA = "correlation-data"; +using OnDataReceivedCallback = std::function; /** * @brief This interface will be used by all objects receiving data from the cloud - * - * The configuration will done by the bootstrap with the implementing class. - * To register an IReceiverCallback use the subscribeToDataReceived method. - * \code{.cpp} - * class ExampleReceiver:IReceiverCallback { - * startReceiving(IReceiver &r) { - * r.subscribeToDataReceived(this); - * } - * onDataReceived( std::uint8_t *buf, size_t size ) { - * // copy buf if needed - * } - * }; - * \endcode - * @see IReceiverCallback */ class IReceiver { public: - ~IReceiver() = default; + virtual ~IReceiver() = default; + /** * @brief indicates if the connection is established and authenticated * diff --git a/src/ISender.h b/src/ISender.h index 4532f1d6..96c8b242 100644 --- a/src/ISender.h +++ b/src/ISender.h @@ -4,6 +4,7 @@ #pragma once #include "IConnectionTypes.h" +#include #include namespace Aws @@ -24,6 +25,21 @@ struct CollectionSchemeParams uint32_t eventID{ 0 }; // event id }; +/** + * @brief called after data is sent + * + * Be cautious this callback will happen from a different thread and the callee + * needs to ensure that the data is treated in a thread safe manner when copying it. + * + * @param result whether the data was successfully sent or not + */ +using OnDataSentCallback = std::function; + +/** + * @brief called after the mqtt client is connected + */ +using OnConnectionEstablishedCallback = std::function; + /** * @brief This interface will be used by all objects sending data to the cloud * @@ -34,6 +50,7 @@ class ISender public: virtual ~ISender() = default; + /** * @brief indicates if the connection is established and authenticated * @@ -62,31 +79,18 @@ class ISender * The data in this buffer is associated with one collectionScheme. * @param size number of accessible bytes in buf. If bigger than getMaxSendSize() this function * will return an error and nothing will be sent. - * @param collectionSchemeParams object containing collectionScheme related metadata for data persistency and - * transmission - * - * @return SUCCESS if connection is established. + * @param callback callback that will be called when the operation completes (successfully or not). + * IMPORTANT: The callback can be called by the same thread before sendBuffer even returns + * or a separate thread, depending on whether the results are known synchronously or asynchronously. */ - virtual ConnectivityError sendBuffer( - const std::uint8_t *buf, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) = 0; + virtual void sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) = 0; - /** - * @brief called to send data from file to the cloud - * - * The function will return after async upload was successfully initiated or error occurred. - * - * @param filePath path to the file to upload - * @param size size of the payload - * @param collectionSchemeParams object containing collectionScheme related metadata for data persistency and - * transmission - * - * @return SUCCESS if connection is established. - */ - virtual ConnectivityError sendFile( const std::string &filePath, - size_t size, - CollectionSchemeParams collectionSchemeParams = CollectionSchemeParams() ) = 0; + virtual void sendBufferToTopic( const std::string &topic, + const std::uint8_t *buf, + size_t size, + OnDataSentCallback callback ) = 0; + + virtual unsigned getPayloadCountSent() const = 0; }; } // namespace IoTFleetWise diff --git a/src/IWaveGpsSource.cpp b/src/IWaveGpsSource.cpp index 21e80031..e182f001 100644 --- a/src/IWaveGpsSource.cpp +++ b/src/IWaveGpsSource.cpp @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 #include "IWaveGpsSource.h" +#include "CollectionInspectionAPITypes.h" #include "LoggingModule.h" -#include "TraceModule.h" +#include "QueueTypes.h" #include #include #include @@ -30,8 +31,8 @@ const std::string DEFAULT_PATH_TO_NMEA_SOURCE = "/dev/" + DEFAULT_NMEA_SOURCE; const std::string SYS_USB_DEVICES_PATH = "/sys/bus/usb/devices"; const std::string QUECTEL_VENDOR_ID = "2c7c"; -static std::string -getFileContents( const std::string &p ) +std::string +IWaveGpsSource::getFileContents( const std::string &p ) { constexpr auto NUM_CHARS = 1; std::string ret; @@ -47,8 +48,8 @@ getFileContents( const std::string &p ) return ret; } -static bool -detectQuectelDevice() +bool +IWaveGpsSource::detectQuectelDevice() { if ( !boost::filesystem::exists( DEFAULT_PATH_TO_NMEA_SOURCE ) ) { @@ -73,8 +74,8 @@ detectQuectelDevice() return false; } -IWaveGpsSource::IWaveGpsSource( SignalBufferPtr signalBufferPtr ) - : mSignalBufferPtr{ std::move( signalBufferPtr ) } +IWaveGpsSource::IWaveGpsSource( SignalBufferDistributorPtr signalBufferDistributor ) + : mSignalBufferDistributor{ std::move( signalBufferDistributor ) } { } @@ -160,27 +161,18 @@ IWaveGpsSource::pollData() } // If values were found pass them on as Signals similar to CAN Signals - if ( foundValid && mSignalBufferPtr != nullptr ) + if ( foundValid && mSignalBufferDistributor != nullptr ) { mValidCoordinateCounter++; auto timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( - CollectedSignal( getSignalIdFromStartBit( mLatitudeStartBit ), timestamp, lastValidLatitude ) ); - collectedSignalsGroup.push_back( - CollectedSignal( getSignalIdFromStartBit( mLongitudeStartBit ), timestamp, lastValidLongitude ) ); - - TraceModule::get().addToAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, 2 ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); + collectedSignalsGroup.push_back( CollectedSignal( + getSignalIdFromStartBit( mLatitudeStartBit ), timestamp, lastValidLatitude, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( + getSignalIdFromStartBit( mLongitudeStartBit ), timestamp, lastValidLongitude, SignalType::DOUBLE ) ); - if ( !mSignalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ) - { - TraceModule::get().subtractFromAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - 2 ); - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - FWE_LOG_WARN( "Signal buffer full" ); - } + mSignalBufferDistributor->push( CollectedDataFrame( collectedSignalsGroup ) ); } if ( mCyclicLoggingTimer.getElapsedMs().count() > CYCLIC_LOG_PERIOD_MS ) { diff --git a/src/IWaveGpsSource.h b/src/IWaveGpsSource.h index 68fb3fc5..96c2abf2 100644 --- a/src/IWaveGpsSource.h +++ b/src/IWaveGpsSource.h @@ -25,9 +25,9 @@ class IWaveGpsSource : public CustomDataSource { public: /** - * @param signalBufferPtr the signal buffer is used pass extracted data + * @param signalBufferDistributor Signal buffer distributor */ - IWaveGpsSource( SignalBufferPtr signalBufferPtr ); + IWaveGpsSource( SignalBufferDistributorPtr signalBufferDistributor ); /** * Initialize IWaveGpsSource and set filter for CustomDataSource * @@ -61,6 +61,8 @@ class IWaveGpsSource : public CustomDataSource private: static bool validLatitude( double latitude ); static bool validLongitude( double longitude ); + static std::string getFileContents( const std::string &p ); + static bool detectQuectelDevice(); /** * The NMEA protocol provides the position in $GPGGA in the following format @@ -82,7 +84,7 @@ class IWaveGpsSource : public CustomDataSource uint16_t mLongitudeStartBit = 0; int mFileHandle = -1; - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; std::shared_ptr mClock = ClockHandler::getClock(); Timer mCyclicLoggingTimer; uint32_t mGpggaLineCounter = 0; diff --git a/src/InspectionMatrixExtractor.cpp b/src/InspectionMatrixExtractor.cpp index f7f66170..0ad048b1 100644 --- a/src/InspectionMatrixExtractor.cpp +++ b/src/InspectionMatrixExtractor.cpp @@ -6,7 +6,7 @@ #include "LoggingModule.h" #include // IWYU pragma: keep #include -#include +#include #include #include // IWYU pragma: keep #include @@ -18,55 +18,57 @@ namespace IoTFleetWise #ifdef FWE_FEATURE_VISION_SYSTEM_DATA void -CollectionSchemeManager::updateRawDataBufferConfig( - std::shared_ptr complexDataDecoderDictionary ) +CollectionSchemeManager::updateRawDataBufferConfigComplexSignals( + std::shared_ptr complexDataDecoderDictionary, + std::unordered_map &updatedSignals ) { - std::unordered_map updatedSignals; - // Iterate through enabled collectionScheme lists to locate the signals and CAN frames to be collected - for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); ++it ) + if ( ( mDecoderManifest != nullptr ) && isCollectionSchemesInSyncWithDm() ) { - const auto &collectionSchemePtr = it->second; - // first iterate through the signalID lists - for ( const auto &signalInfo : collectionSchemePtr->getCollectSignals() ) + // Iterate through enabled collectionScheme lists to locate the signals and CAN frames to be collected + for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); ++it ) { - SignalID signalId = signalInfo.signalID; - if ( ( signalId & INTERNAL_SIGNAL_ID_BITMASK ) != 0 ) + const auto &collectionSchemePtr = it->second; + // first iterate through the signalID lists + for ( const auto &signalInfo : collectionSchemePtr->getCollectSignals() ) { - continue; - } + SignalID signalId = signalInfo.signalID; + if ( ( signalId & INTERNAL_SIGNAL_ID_BITMASK ) != 0 ) + { + continue; + } - auto networkType = mDecoderManifest->getNetworkProtocol( signalId ); - if ( networkType != VehicleDataSourceProtocol::COMPLEX_DATA ) - { - continue; - } + auto networkType = mDecoderManifest->getNetworkProtocol( signalId ); + if ( networkType != VehicleDataSourceProtocol::COMPLEX_DATA ) + { + continue; + } - RawData::SignalUpdateConfig signalConfig; - signalConfig.typeId = signalId; + RawData::SignalUpdateConfig signalConfig; + signalConfig.typeId = signalId; - auto complexSignalDecoderFormat = mDecoderManifest->getComplexSignalDecoderFormat( signalId ); - signalConfig.interfaceId = complexSignalDecoderFormat.mInterfaceId; - if ( complexDataDecoderDictionary != nullptr ) - { - auto interface = complexDataDecoderDictionary->complexMessageDecoderMethod.find( - complexSignalDecoderFormat.mInterfaceId ); - // Now try to find the messageId for this signal - if ( interface != complexDataDecoderDictionary->complexMessageDecoderMethod.end() ) + auto complexSignalDecoderFormat = mDecoderManifest->getComplexSignalDecoderFormat( signalId ); + signalConfig.interfaceId = complexSignalDecoderFormat.mInterfaceId; + if ( complexDataDecoderDictionary != nullptr ) { - auto complexDataMessage = std::find_if( - interface->second.begin(), interface->second.end(), [signalId]( const auto &pair ) -> bool { - return pair.second.mSignalId == signalId; - } ); - if ( complexDataMessage != interface->second.end() ) + auto interface = complexDataDecoderDictionary->complexMessageDecoderMethod.find( + complexSignalDecoderFormat.mInterfaceId ); + // Now try to find the messageId for this signal + if ( interface != complexDataDecoderDictionary->complexMessageDecoderMethod.end() ) { - signalConfig.messageId = complexDataMessage->first; + auto complexDataMessage = std::find_if( + interface->second.begin(), interface->second.end(), [signalId]( const auto &pair ) -> bool { + return pair.second.mSignalId == signalId; + } ); + if ( complexDataMessage != interface->second.end() ) + { + signalConfig.messageId = complexDataMessage->first; + } } } + updatedSignals[signalId] = signalConfig; } - updatedSignals[signalId] = signalConfig; } } - mRawDataBufferManager->updateConfig( updatedSignals ); } #endif @@ -74,10 +76,11 @@ void CollectionSchemeManager::addConditionData( const ICollectionSchemePtr &collectionScheme, ConditionWithCollectedData &conditionData ) { - conditionData.minimumPublishIntervalMs = collectionScheme->getMinimumPublishIntervalMs(); - conditionData.afterDuration = collectionScheme->getAfterDurationMs(); - conditionData.includeActiveDtcs = collectionScheme->isActiveDTCsIncluded(); - conditionData.triggerOnlyOnRisingEdge = collectionScheme->isTriggerOnlyOnRisingEdge(); + conditionData.metadata.compress = collectionScheme->isCompressionNeeded(); + conditionData.metadata.persist = collectionScheme->isPersistNeeded(); + conditionData.metadata.priority = collectionScheme->getPriority(); + conditionData.metadata.decoderID = collectionScheme->getDecoderManifestID(); + conditionData.metadata.collectionSchemeID = collectionScheme->getCollectionSchemeID(); /* * use for loop to copy signalInfo and CANframe over to avoid error or memory issue @@ -97,120 +100,153 @@ CollectionSchemeManager::addConditionData( const ICollectionSchemePtr &collectio conditionData.signals.emplace_back( inspectionSignal ); } - const std::vector &collectionCANFrames = collectionScheme->getCollectRawCanFrames(); - for ( uint32_t i = 0; i < collectionCANFrames.size(); i++ ) { - InspectionMatrixCanFrameCollectionInfo CANFrame = {}; - CANFrame.frameID = collectionCANFrames[i].frameID; - CANFrame.channelID = mCANIDTranslator.getChannelNumericID( collectionCANFrames[i].interfaceID ); - CANFrame.sampleBufferSize = collectionCANFrames[i].sampleBufferSize; - CANFrame.minimumSampleIntervalMs = collectionCANFrames[i].minimumSampleIntervalMs; - if ( CANFrame.channelID == INVALID_CAN_SOURCE_NUMERIC_ID ) - { - FWE_LOG_WARN( "Invalid Interface ID provided: " + collectionCANFrames[i].interfaceID ); - } - else + const std::vector &collectionCANFrames = collectionScheme->getCollectRawCanFrames(); + for ( uint32_t i = 0; i < collectionCANFrames.size(); i++ ) { - conditionData.canFrames.emplace_back( CANFrame ); + InspectionMatrixCanFrameCollectionInfo CANFrame = {}; + CANFrame.frameID = collectionCANFrames[i].frameID; + CANFrame.channelID = mCANIDTranslator.getChannelNumericID( collectionCANFrames[i].interfaceID ); + CANFrame.sampleBufferSize = collectionCANFrames[i].sampleBufferSize; + CANFrame.minimumSampleIntervalMs = collectionCANFrames[i].minimumSampleIntervalMs; + if ( CANFrame.channelID == INVALID_CAN_SOURCE_NUMERIC_ID ) + { + FWE_LOG_WARN( "Invalid Interface ID provided: " + collectionCANFrames[i].interfaceID ); + } + else + { + conditionData.canFrames.emplace_back( CANFrame ); + } } + + conditionData.minimumPublishIntervalMs = collectionScheme->getMinimumPublishIntervalMs(); + conditionData.afterDuration = collectionScheme->getAfterDurationMs(); + conditionData.includeActiveDtcs = collectionScheme->isActiveDTCsIncluded(); + conditionData.triggerOnlyOnRisingEdge = collectionScheme->isTriggerOnlyOnRisingEdge(); } - // The rest - conditionData.metadata.compress = collectionScheme->isCompressionNeeded(); - conditionData.metadata.persist = collectionScheme->isPersistNeeded(); - conditionData.metadata.priority = collectionScheme->getPriority(); - conditionData.metadata.decoderID = collectionScheme->getDecoderManifestID(); - conditionData.metadata.collectionSchemeID = collectionScheme->getCollectionSchemeID(); +} + +static void +buildExpressionNodeMapAndVector( const ExpressionNode *expressionNode, + std::map &expressionNodeToIndexMap, + std::vector &expressionNodes, + uint32_t &index, + bool &isStaticCondition, + bool &alwaysEvaluateCondition ) +{ + if ( expressionNode == nullptr ) + { + return; + } + + expressionNodeToIndexMap[expressionNode] = index; + index++; + expressionNodes.push_back( expressionNode ); + + if ( expressionNode->nodeType == ExpressionNodeType::SIGNAL ) + { + // If at least one of the nodes is a signal value, condition is not static + isStaticCondition = false; + } + + buildExpressionNodeMapAndVector( expressionNode->left, + expressionNodeToIndexMap, + expressionNodes, + index, + isStaticCondition, + alwaysEvaluateCondition ); + buildExpressionNodeMapAndVector( expressionNode->right, + expressionNodeToIndexMap, + expressionNodes, + index, + isStaticCondition, + alwaysEvaluateCondition ); } void -CollectionSchemeManager::inspectionMatrixExtractor( const std::shared_ptr &inspectionMatrix ) +CollectionSchemeManager::matrixExtractor( const std::shared_ptr &inspectionMatrix ) { - std::stack nodeStack; - std::map nodeToIndexMap; - std::vector nodes; - uint32_t index = 0; - const ExpressionNode *currNode = nullptr; + std::map expressionNodeToIndexMap; + std::vector expressionNodes; + uint32_t index = 0U; - for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); it++ ) + if ( !isCollectionSchemesInSyncWithDm() ) { - ICollectionSchemePtr collectionScheme = it->second; - ConditionWithCollectedData conditionData; - addConditionData( collectionScheme, conditionData ); - - currNode = collectionScheme->getCondition(); - /* save the old root of this tree */ - conditionData.condition = currNode; - inspectionMatrix->conditions.emplace_back( conditionData ); - - /* - * The following lines traverse each tree and pack the node addresses into a vector - * and build a map - * any order to traverse the tree is OK, here we use in-order. - */ - while ( currNode != nullptr ) - { - nodeStack.push( currNode ); - currNode = currNode->left; - } - while ( !nodeStack.empty() ) - { - currNode = nodeStack.top(); - nodeStack.pop(); - nodeToIndexMap[currNode] = index; - nodes.emplace_back( currNode ); - index++; - if ( currNode->right != nullptr ) - { - currNode = currNode->right; - while ( currNode != nullptr ) - { - nodeStack.push( currNode ); - currNode = currNode->left; - } - } - } + return; } - size_t count = nodes.size(); - /* now we have the count of all nodes from all collectionSchemes, allocate a vector for the output */ - inspectionMatrix->expressionNodeStorage.resize( count ); - /* copy from the old tree node and update left and right children pointers */ - for ( uint32_t i = 0; i < count; i++ ) + for ( const auto &enabledCollectionScheme : mEnabledCollectionSchemeMap ) { - inspectionMatrix->expressionNodeStorage[i].nodeType = nodes[i]->nodeType; - inspectionMatrix->expressionNodeStorage[i].floatingValue = nodes[i]->floatingValue; - inspectionMatrix->expressionNodeStorage[i].booleanValue = nodes[i]->booleanValue; - inspectionMatrix->expressionNodeStorage[i].signalID = nodes[i]->signalID; - inspectionMatrix->expressionNodeStorage[i].function = nodes[i]->function; + ICollectionSchemePtr collectionScheme = enabledCollectionScheme.second; + + extractCondition( inspectionMatrix, + collectionScheme, + expressionNodes, + expressionNodeToIndexMap, + index, + collectionScheme->getCondition() ); + } + + // re-build all ExpressionNodes (for storage) and set ExpressionNode pointer addresses appropriately + std::size_t expressionNodeCount = expressionNodes.size(); - if ( nodes[i]->left != nullptr ) + inspectionMatrix->expressionNodeStorage.resize( expressionNodeCount ); + + for ( std::size_t i = 0U; i < expressionNodeCount; i++ ) + { + inspectionMatrix->expressionNodeStorage[i].nodeType = expressionNodes[i]->nodeType; + inspectionMatrix->expressionNodeStorage[i].floatingValue = expressionNodes[i]->floatingValue; + inspectionMatrix->expressionNodeStorage[i].booleanValue = expressionNodes[i]->booleanValue; + inspectionMatrix->expressionNodeStorage[i].signalID = expressionNodes[i]->signalID; + inspectionMatrix->expressionNodeStorage[i].function.windowFunction = + expressionNodes[i]->function.windowFunction; + if ( expressionNodes[i]->left != nullptr ) { - uint32_t leftIndex = nodeToIndexMap[nodes[i]->left]; + uint32_t leftIndex = expressionNodeToIndexMap[expressionNodes[i]->left]; inspectionMatrix->expressionNodeStorage[i].left = &inspectionMatrix->expressionNodeStorage[leftIndex]; } - else - { - inspectionMatrix->expressionNodeStorage[i].left = nullptr; - } - if ( nodes[i]->right != nullptr ) + if ( expressionNodes[i]->right != nullptr ) { - uint32_t rightIndex = nodeToIndexMap[nodes[i]->right]; + uint32_t rightIndex = expressionNodeToIndexMap[expressionNodes[i]->right]; inspectionMatrix->expressionNodeStorage[i].right = &inspectionMatrix->expressionNodeStorage[rightIndex]; } - else - { - inspectionMatrix->expressionNodeStorage[i].right = nullptr; - } } - /* update the root of tree with new address */ - for ( uint32_t i = 0; i < inspectionMatrix->conditions.size(); i++ ) + + for ( auto &conditionWithCollectedData : inspectionMatrix->conditions ) { - uint32_t newIndex = nodeToIndexMap[inspectionMatrix->conditions[i].condition]; - inspectionMatrix->conditions[i].condition = &inspectionMatrix->expressionNodeStorage[newIndex]; + if ( conditionWithCollectedData.condition != nullptr ) + { + uint32_t conditionIndex = expressionNodeToIndexMap[conditionWithCollectedData.condition]; + + conditionWithCollectedData.condition = &inspectionMatrix->expressionNodeStorage[conditionIndex]; + } } } +void +CollectionSchemeManager::extractCondition( const std::shared_ptr &inspectionMatrix, + const ICollectionSchemePtr &collectionScheme, + std::vector &nodes, + std::map &nodeToIndexMap, + uint32_t &index, + const ExpressionNode *initialNode ) +{ + ConditionWithCollectedData conditionData{}; + addConditionData( collectionScheme, conditionData ); + const ExpressionNode *currNode = initialNode; + /* save the old root of this tree */ + conditionData.condition = currNode; + + buildExpressionNodeMapAndVector( currNode, + nodeToIndexMap, + nodes, + index, + conditionData.isStaticCondition, + conditionData.alwaysEvaluateCondition ); + inspectionMatrix->conditions.emplace_back( conditionData ); +} + void CollectionSchemeManager::inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) { diff --git a/src/IoTFleetWiseConfig.cpp b/src/IoTFleetWiseConfig.cpp index c6570f8b..ff5bc892 100644 --- a/src/IoTFleetWiseConfig.cpp +++ b/src/IoTFleetWiseConfig.cpp @@ -4,7 +4,6 @@ #include "IoTFleetWiseConfig.h" #include #include -#include #include namespace Aws @@ -46,11 +45,13 @@ IoTFleetWiseConfig::asStringRequired() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( !mConfig.isString() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a string at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a string at " + mPath ); } return mConfig.asString(); } @@ -64,7 +65,8 @@ IoTFleetWiseConfig::asStringOptional() const } if ( !mConfig.isString() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a string at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a string at " + mPath ); } return boost::make_optional( mConfig.asString() ); } @@ -74,11 +76,13 @@ IoTFleetWiseConfig::asU32Required() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( !mConfig.isUInt() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid uint32 at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid uint32 at " + mPath ); } return mConfig.asUInt(); } @@ -92,7 +96,8 @@ IoTFleetWiseConfig::asU32Optional() const } if ( !mConfig.isUInt() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid uint32 at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid uint32 at " + mPath ); } return boost::make_optional( mConfig.asUInt() ); } @@ -102,11 +107,13 @@ IoTFleetWiseConfig::asU64Required() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( !mConfig.isUInt64() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid uint64 at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid uint64 at " + mPath ); } return mConfig.asUInt64(); } @@ -120,7 +127,8 @@ IoTFleetWiseConfig::asU64Optional() const } if ( !mConfig.isUInt64() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid uint64 at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid uint64 at " + mPath ); } return boost::make_optional( mConfig.asUInt64() ); } @@ -128,6 +136,7 @@ IoTFleetWiseConfig::asU64Optional() const uint32_t IoTFleetWiseConfig::asU32FromStringRequired() const { + // coverity[fun_call_w_exception] False positive, if this function is not called, there won't be a catch for it auto value = asStringRequired(); try { @@ -135,13 +144,15 @@ IoTFleetWiseConfig::asU32FromStringRequired() const } catch ( ... ) { - throw std::runtime_error( "Could not convert '" + value + "' to uint32 for config value at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Could not convert '" + value + "' to uint32 for config value at " + mPath ); } } boost::optional IoTFleetWiseConfig::asU32FromStringOptional() const { + // coverity[fun_call_w_exception] False positive, if this function is not called, there won't be a catch for it auto value = asStringOptional(); if ( !value.has_value() ) { @@ -153,7 +164,9 @@ IoTFleetWiseConfig::asU32FromStringOptional() const } catch ( ... ) { - throw std::runtime_error( "Could not convert '" + value.get() + "' to uint32 for config value at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, + "Could not convert '" + value.get() + "' to uint32 for config value at " + mPath ); } } @@ -162,11 +175,13 @@ IoTFleetWiseConfig::asSizeRequired() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( sizeof( size_t ) >= sizeof( uint64_t ) ? !mConfig.isUInt64() : !mConfig.isUInt() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid size at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid size at " + mPath ); } return sizeof( size_t ) >= sizeof( uint64_t ) ? mConfig.asUInt64() : mConfig.asUInt(); } @@ -180,7 +195,8 @@ IoTFleetWiseConfig::asSizeOptional() const } if ( sizeof( size_t ) >= sizeof( uint64_t ) ? !mConfig.isUInt64() : !mConfig.isUInt() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a valid size at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a valid size at " + mPath ); } return boost::make_optional( static_cast( sizeof( size_t ) >= sizeof( uint64_t ) ? mConfig.asUInt64() : mConfig.asUInt() ) ); @@ -191,11 +207,13 @@ IoTFleetWiseConfig::asBoolRequired() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( !mConfig.isBool() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a bool at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a bool at " + mPath ); } return mConfig.asBool(); } @@ -209,7 +227,8 @@ IoTFleetWiseConfig::asBoolOptional() const } if ( !mConfig.isBool() ) { - throw std::runtime_error( "Config value " + getValueString() + "is not a bool at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value " + getValueString() + "is not a bool at " + mPath ); } return boost::make_optional( mConfig.asBool() ); } @@ -225,11 +244,13 @@ IoTFleetWiseConfig::getArraySizeRequired() const { if ( mConfig.isNull() ) { - throw std::runtime_error( "Config value missing at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value missing at " + mPath ); } if ( !mConfig.isArray() ) { - throw std::runtime_error( "Config value is not an array at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value is not an array at " + mPath ); } return mConfig.size(); } @@ -243,7 +264,8 @@ IoTFleetWiseConfig::getArraySizeOptional() const } if ( !mConfig.isArray() ) { - throw std::runtime_error( "Config value is not an array at " + mPath ); + // coverity[exception_thrown] False positive, if this function is not called, there won't be a catch for it + throw runtimeError( __LINE__, "Config value is not an array at " + mPath ); } return mConfig.size(); } diff --git a/src/IoTFleetWiseConfig.h b/src/IoTFleetWiseConfig.h index 932cea7d..f5128a78 100644 --- a/src/IoTFleetWiseConfig.h +++ b/src/IoTFleetWiseConfig.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace Aws @@ -54,6 +55,16 @@ class IoTFleetWiseConfig const Json::Value &mConfig; std::string mPath; std::string getValueString() const; + + // Suppress false positive for Coverity autosar_cpp14_a15_1_3_violation - all exceptions thrown are uniquely + // identified by the config path + static inline std::runtime_error + runtimeError( int line, const std::string &message ) + { + // Ignore the line, passing it convinces Coverity that this is a unique exception + static_cast( line ); + return std::runtime_error( message ); + } }; } // namespace IoTFleetWise diff --git a/src/IoTFleetWiseEngine.cpp b/src/IoTFleetWiseEngine.cpp index cae09fbb..d6a35059 100644 --- a/src/IoTFleetWiseEngine.cpp +++ b/src/IoTFleetWiseEngine.cpp @@ -7,12 +7,16 @@ #include "AwsSDKMemoryManager.h" #include "CollectionInspectionAPITypes.h" #include "DataSenderManager.h" +#include "DataSenderProtoWriter.h" +#include "DataSenderTypes.h" #include "ILogger.h" #include "IoTFleetWiseConfig.h" #include "LogLevel.h" #include "LoggingModule.h" #include "MqttClientWrapper.h" +#include "QueueTypes.h" #include "SignalTypes.h" +#include "TelemetryDataSender.h" #include "TraceModule.h" #include #include @@ -23,24 +27,27 @@ #include #include #include +#include #include #ifdef FWE_FEATURE_GREENGRASSV2 -#include "AwsGGConnectivityModule.h" -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +#include "AwsGreengrassV2ConnectivityModule.h" +#ifdef FWE_FEATURE_S3 #include #endif #endif -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +#ifdef FWE_FEATURE_S3 #include "Credentials.h" -#include "DataSenderIonWriter.h" -#include "RawDataManager.h" #include "TransferManagerWrapper.h" #include #include #include #include #endif +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +#include "DataSenderIonWriter.h" +#include "VisionSystemDataSender.h" +#endif namespace Aws { @@ -90,8 +97,39 @@ getFileContents( const std::string &p ) return ret; } +/** + * @brief Get the absolute file path, if the path is already absolute its returned. + * + * @param p The file path + * @param basePath Base path to which the p is relative + * @return boost::filesystem::path Absolute file path + */ +boost::filesystem::path +getAbsolutePath( const std::string &p, const boost::filesystem::path &basePath ) +{ + boost::filesystem::path filePath( p ); + if ( !filePath.is_absolute() ) + { + return basePath / filePath; + } + return filePath; +} + } // namespace +#ifdef FWE_FEATURE_S3 +std::shared_ptr +IoTFleetWiseEngine::getTransferManagerExecutor() +{ + std::lock_guard lock( mTransferManagerExecutorMutex ); + if ( mTransferManagerExecutor == nullptr ) + { + mTransferManagerExecutor = Aws::MakeShared( "executor", 25 ); + } + return mTransferManagerExecutor; +} +#endif + IoTFleetWiseEngine::IoTFleetWiseEngine() { TraceModule::get().sectionBegin( TraceSection::FWE_STARTUP ); @@ -108,27 +146,35 @@ IoTFleetWiseEngine::~IoTFleetWiseEngine() } bool -IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) +IoTFleetWiseEngine::connect( const Json::Value &jsonConfig, const boost::filesystem::path &configFileDirectoryPath ) { // Main bootstrap sequence. try { IoTFleetWiseConfig config( jsonConfig ); - const auto persistencyPath = config["staticConfig"]["persistency"]["persistencyPath"].asStringRequired(); - /*************************Payload Manager and Persistency library bootstrap begin*********/ - // Create an object for Persistency - mPersistDecoderManifestCollectionSchemesAndData = std::make_shared( - persistencyPath, config["staticConfig"]["persistency"]["persistencyPartitionMaxSize"].asSizeRequired() ); - if ( !mPersistDecoderManifestCollectionSchemesAndData->init() ) + uint64_t persistencyUploadRetryIntervalMs = 0; + if ( ( config["staticConfig"].isMember( "persistency" ) ) ) { - FWE_LOG_ERROR( "Failed to init persistency library" ); + const auto persistencyPath = config["staticConfig"]["persistency"]["persistencyPath"].asStringRequired(); + /*************************Payload Manager and Persistency library bootstrap begin*********/ + // Create an object for Persistency + mPersistDecoderManifestCollectionSchemesAndData = std::make_shared( + getAbsolutePath( persistencyPath, configFileDirectoryPath ).string(), + config["staticConfig"]["persistency"]["persistencyPartitionMaxSize"].asSizeRequired() ); + if ( !mPersistDecoderManifestCollectionSchemesAndData->init() ) + { + FWE_LOG_ERROR( "Failed to init persistency library" ); + } + persistencyUploadRetryIntervalMs = + config["staticConfig"]["persistency"]["persistencyUploadRetryIntervalMs"].asU64Optional().get_value_or( + DEFAULT_RETRY_UPLOAD_PERSISTED_INTERVAL_MS ); + // Payload Manager for offline data management + mPayloadManager = std::make_shared( mPersistDecoderManifestCollectionSchemesAndData ); + } + else + { + FWE_LOG_INFO( "Persistency feature is disabled in the configuration." ); } - uint64_t persistencyUploadRetryIntervalMs = - config["staticConfig"]["persistency"]["persistencyUploadRetryIntervalMs"].asU64Optional().get_value_or( - DEFAULT_RETRY_UPLOAD_PERSISTED_INTERVAL_MS ); - // Payload Manager for offline data management - mPayloadManager = std::make_shared( mPersistDecoderManifestCollectionSchemesAndData ); - /*************************Payload Manager and Persistency library bootstrap end************/ /*************************CAN InterfaceID to InternalID Translator begin*********/ @@ -219,7 +265,10 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } else if ( mqttConfig.isMember( "privateKeyFilename" ) ) { - privateKey = getFileContents( mqttConfig["privateKeyFilename"].asStringRequired() ); + auto privKeyPathAbs = + getAbsolutePath( mqttConfig["privateKeyFilename"].asStringRequired(), configFileDirectoryPath ) + .string(); + privateKey = getFileContents( privKeyPathAbs ); } if ( mqttConfig.isMember( "certificate" ) ) { @@ -227,7 +276,10 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } else if ( mqttConfig.isMember( "certificateFilename" ) ) { - certificate = getFileContents( mqttConfig["certificateFilename"].asStringRequired() ); + auto certPathAbs = + getAbsolutePath( mqttConfig["certificateFilename"].asStringRequired(), configFileDirectoryPath ) + .string(); + certificate = getFileContents( certPathAbs ); } if ( mqttConfig.isMember( "rootCA" ) ) { @@ -235,7 +287,10 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } else if ( mqttConfig.isMember( "rootCAFilename" ) ) { - rootCA = getFileContents( mqttConfig["rootCAFilename"].asStringRequired() ); + auto rootCAPathAbs = + getAbsolutePath( mqttConfig["rootCAFilename"].asStringRequired(), configFileDirectoryPath ) + .string(); + rootCA = getFileContents( rootCAPathAbs ); } // coverity[autosar_cpp14_a20_8_5_violation] - can't use make_unique as the constructor is private auto builder = std::unique_ptr( @@ -258,10 +313,20 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) builderWrapper = std::make_unique( std::move( builder ) ); } - mConnectivityModule = - std::make_shared( rootCA, clientId, std::move( builderWrapper ) ); + AwsIotConnectivityConfig mqttConnectionConfig; + mqttConnectionConfig.keepAliveIntervalSeconds = + static_cast( mqttConfig["keepAliveIntervalSeconds"].asU32Optional().get_value_or( + MQTT_KEEP_ALIVE_INTERVAL_SECONDS ) ); + mqttConnectionConfig.pingTimeoutMs = + mqttConfig["pingTimeoutMs"].asU32Optional().get_value_or( MQTT_PING_TIMEOUT_MS ); + mqttConnectionConfig.sessionExpiryIntervalSeconds = + mqttConfig["sessionExpiryIntervalSeconds"].asU32Optional().get_value_or( + MQTT_SESSION_EXPIRY_INTERVAL_SECONDS ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + mConnectivityModule = std::make_shared( + rootCA, clientId, std::move( builderWrapper ), mqttConnectionConfig ); + +#ifdef FWE_FEATURE_S3 if ( config["staticConfig"].isMember( "credentialsProvider" ) ) { auto crtCredentialsProvider = createX509CredentialsProvider( @@ -279,8 +344,8 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) else if ( connectionType == "iotGreengrassV2" ) { FWE_LOG_INFO( "ConnectionType is iotGreengrassV2" ); - mConnectivityModule = std::make_shared( bootstrapPtr ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + mConnectivityModule = std::make_shared( bootstrapPtr ); +#ifdef FWE_FEATURE_S3 mAwsCredentialsProvider = std::make_shared(); #endif } @@ -293,41 +358,37 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) // Only CAN data channel needs a payloadManager object for persistency and compression support, // for other components this will be nullptr - mConnectivityChannelSendVehicleData = - mConnectivityModule->createNewChannel( mPayloadManager, mqttConfig["canDataTopic"].asStringRequired() ); + mSenderVehicleData = mConnectivityModule->createSender( mqttConfig["canDataTopic"].asStringRequired() ); - mConnectivityChannelReceiveCollectionSchemeList = mConnectivityModule->createNewChannel( - nullptr, mqttConfig["collectionSchemeListTopic"].asStringRequired(), true ); + mReceiverCollectionSchemeList = + mConnectivityModule->createReceiver( mqttConfig["collectionSchemeListTopic"].asStringRequired() ); - mConnectivityChannelReceiveDecoderManifest = mConnectivityModule->createNewChannel( - nullptr, mqttConfig["decoderManifestTopic"].asStringRequired(), true ); + mReceiverDecoderManifest = + mConnectivityModule->createReceiver( mqttConfig["decoderManifestTopic"].asStringRequired() ); /* - * Over this channel metrics like performance (resident ram pages, cpu time spent in threads) + * Over this sender, metrics like performance (resident ram pages, cpu time spent in threads) * and tracing metrics like internal variables and time spent in specially instrumented functions * spent are uploaded in json format over mqtt. */ auto metricsUploadTopic = mqttConfig["metricsUploadTopic"].asStringOptional().get_value_or( "" ); if ( !metricsUploadTopic.empty() ) { - mConnectivityChannelMetricsUpload = mConnectivityModule->createNewChannel( nullptr, metricsUploadTopic ); + mSenderMetrics = mConnectivityModule->createSender( metricsUploadTopic ); } /* - * Over this channel log messages that are currently logged to STDOUT are uploaded in json + * Over this sender, log messages that are currently logged to STDOUT are uploaded in json * format over MQTT. */ auto loggingUploadTopic = mqttConfig["loggingUploadTopic"].asStringOptional().get_value_or( "" ); if ( !loggingUploadTopic.empty() ) { - mConnectivityChannelLogsUpload = mConnectivityModule->createNewChannel( nullptr, loggingUploadTopic ); + mSenderLogs = mConnectivityModule->createSender( loggingUploadTopic ); } // Create an ISender for sending Checkins - mConnectivityChannelSendCheckin = - mConnectivityModule->createNewChannel( nullptr, mqttConfig["checkinTopic"].asStringRequired() ); + mSenderCheckin = mConnectivityModule->createSender( mqttConfig["checkinTopic"].asStringRequired() ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - std::shared_ptr rawDataBufferManager; boost::optional rawDataBufferManagerConfig; auto rawDataBufferJsonConfig = config["staticConfig"]["visionSystemDataCollection"]["rawDataBuffer"]; auto rawBufferSize = rawDataBufferJsonConfig["maxSize"].asSizeOptional(); @@ -360,15 +421,9 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) FWE_LOG_ERROR( "Failed to create raw data buffer manager config" ); return false; } - rawDataBufferManager = std::make_shared( rawDataBufferManagerConfig.get() ); + mRawBufferManager = std::make_shared( rawDataBufferManagerConfig.get() ); } -#endif - // For asynchronous connect the call needs to be done after all channels created and setTopic calls - if ( !mConnectivityModule->connect() ) - { - return false; - } /*************************Connectivity bootstrap end***************************************/ /*************************Remote Profiling bootstrap begin**********************************/ @@ -401,53 +456,82 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) // loggingUploadMaxWaitBeforeUploadMs // profilerPrefix mRemoteProfiler = std::make_unique( - mConnectivityChannelMetricsUpload, - mConnectivityChannelLogsUpload, + mSenderMetrics, + mSenderLogs, config["staticConfig"]["remoteProfilerDefaultValues"]["metricsUploadIntervalMs"].asU32Required(), config["staticConfig"]["remoteProfilerDefaultValues"]["loggingUploadMaxWaitBeforeUploadMs"] .asU32Required(), logThreshold, config["staticConfig"]["remoteProfilerDefaultValues"]["profilerPrefix"].asStringRequired() ); - if ( !mRemoteProfiler->start() ) - { - FWE_LOG_WARN( - - "Failed to start the Remote Profiler - No remote profiling available until FWE restart" ); - } setLogForwarding( mRemoteProfiler.get() ); } /*************************Remote Profiling bootstrap ends**********************************/ /*************************Inspection Engine bootstrap begin*********************************/ - // Below are three buffers to be shared between Vehicle Data Consumer and Collection Engine - // Signal Buffer are a lock-free multi-producer single consumer buffer - auto signalBufferPtr = std::make_shared( - config["staticConfig"]["bufferSizes"]["decodedSignalsBufferSize"].asSizeRequired() ); + auto signalBufferSize = config["staticConfig"]["bufferSizes"]["decodedSignalsBufferSize"].asSizeRequired(); + auto signalBuffer = + std::make_shared( signalBufferSize, + "Signal Buffer", + TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES, + // Notify listeners when 10% of the buffer is full so that we don't + // let it grow too much. + signalBufferSize / 10 ); + + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + // Create the Data Inspection Queue - mCollectedDataReadyToPublish = std::make_shared( - config["staticConfig"]["internalParameters"]["readyToPublishDataBufferSize"].asSizeRequired() ); + mCollectedDataReadyToPublish = std::make_shared( + config["staticConfig"]["internalParameters"]["readyToPublishDataBufferSize"].asSizeRequired(), + "Collected Data", + TraceAtomicVariable::QUEUE_INSPECTION_TO_SENDER ); + + auto dataSenderProtoWriter = std::make_shared( mCANIDTranslator, mRawBufferManager ); + auto payloadConfigUncompressed = config["staticConfig"]["payloadAdaption"]["uncompressed"]; + PayloadAdaptionConfig payloadAdaptionConfigUncompressed{ + payloadConfigUncompressed["transmitThresholdStartPercent"].asU32Optional().get_value_or( 80 ), + payloadConfigUncompressed["payloadSizeLimitMinPercent"].asU32Optional().get_value_or( 70 ), + payloadConfigUncompressed["payloadSizeLimitMaxPercent"].asU32Optional().get_value_or( 90 ), + payloadConfigUncompressed["transmitThresholdAdaptPercent"].asU32Optional().get_value_or( 10 ) }; + auto payloadConfigCompressed = config["staticConfig"]["payloadAdaption"]["compressed"]; + // Snappy typically compresses to around 30% of original size, so set the starting compressed transmit threshold + // to double the maximum payload size: + PayloadAdaptionConfig payloadAdaptionConfigCompressed{ + payloadConfigCompressed["transmitThresholdStartPercent"].asU32Optional().get_value_or( 200 ), + payloadConfigCompressed["payloadSizeLimitMinPercent"].asU32Optional().get_value_or( 70 ), + payloadConfigCompressed["payloadSizeLimitMaxPercent"].asU32Optional().get_value_or( 90 ), + payloadConfigCompressed["transmitThresholdAdaptPercent"].asU32Optional().get_value_or( 10 ) }; + auto telemetryDataSender = std::make_shared( mSenderVehicleData, + dataSenderProtoWriter, + payloadAdaptionConfigUncompressed, + payloadAdaptionConfigCompressed ); + std::unordered_map> dataSenders; + dataSenders[SenderDataType::TELEMETRY] = telemetryDataSender; // Init and start the Inspection Engine - mCollectionInspectionWorkerThread = std::make_shared(); + mCollectionInspectionEngine = std::make_shared(); + mCollectionInspectionWorkerThread = + std::make_shared( *mCollectionInspectionEngine ); if ( ( !mCollectionInspectionWorkerThread->init( - signalBufferPtr, + signalBuffer, mCollectedDataReadyToPublish, - config["staticConfig"]["threadIdleTimes"]["inspectionThreadIdleTimeMs"].asU32Required() -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - rawDataBufferManager -#endif - ) ) || + config["staticConfig"]["threadIdleTimes"]["inspectionThreadIdleTimeMs"].asU32Required(), + mRawBufferManager ) ) || ( !mCollectionInspectionWorkerThread->start() ) ) { FWE_LOG_ERROR( "Failed to init and start the Inspection Engine" ); return false; } + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda + signalBuffer->subscribeToNewDataAvailable( std::bind( &CollectionInspectionWorkerThread::onNewDataAvailable, + mCollectionInspectionWorkerThread.get() ) ); /*************************Inspection Engine bootstrap end***********************************/ /*************************DataSender bootstrap begin*********************************/ #ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::shared_ptr ionWriter; + std::shared_ptr visionSystemDataSender; if ( ( mAwsCredentialsProvider == nullptr ) || ( !config["staticConfig"].isMember( "s3Upload" ) ) ) { FWE_LOG_INFO( "S3 sender not initialized so no vision-system-data data upload will be supported. Add " @@ -462,9 +546,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) Aws::Transfer::TransferManagerConfiguration &transferManagerConfiguration ) -> std::shared_ptr { clientConfiguration.maxConnections = s3MaxConnections; - mTransferManagerExecutor = - Aws::MakeShared( "executor", 25 ); - transferManagerConfiguration.transferExecutor = mTransferManagerExecutor.get(); + transferManagerConfiguration.transferExecutor = getTransferManagerExecutor().get(); auto s3Client = std::make_shared( mAwsCredentialsProvider, Aws::MakeShared( "S3Client" ), @@ -473,34 +555,28 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) return std::make_shared( Aws::Transfer::TransferManager::Create( transferManagerConfiguration ) ); }; - mS3Sender = - std::make_shared( mPayloadManager, - createTransferManagerWrapper, - config["staticConfig"]["s3Upload"]["multipartSize"].asSizeRequired() ); + mS3Sender = std::make_shared( + createTransferManagerWrapper, config["staticConfig"]["s3Upload"]["multipartSize"].asSizeRequired() ); + ionWriter = std::make_shared( mRawBufferManager, clientId ); + visionSystemDataSender = std::make_shared( + mCollectedDataReadyToPublish, mS3Sender, ionWriter, clientId ); + dataSenders[SenderDataType::VISION_SYSTEM] = visionSystemDataSender; } - auto ionWriter = std::make_shared( rawDataBufferManager, clientId ); #endif - mDataSenderManager = std::make_shared( - mConnectivityChannelSendVehicleData, - mPayloadManager, - mCANIDTranslator, - config["staticConfig"]["publishToCloudParameters"]["maxPublishMessageCount"].asU32Required() -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - mS3Sender, - ionWriter, - clientId -#endif - ); + + mDataSenderManager = + std::make_shared( std::move( dataSenders ), mSenderVehicleData, mPayloadManager ); + std::vector> dataToSendQueues = { mCollectedDataReadyToPublish }; mDataSenderManagerWorkerThread = std::make_shared( - mConnectivityModule, mDataSenderManager, persistencyUploadRetryIntervalMs, mCollectedDataReadyToPublish ); + mConnectivityModule, mDataSenderManager, persistencyUploadRetryIntervalMs, dataToSendQueues ); if ( !mDataSenderManagerWorkerThread->start() ) { FWE_LOG_ERROR( "Failed to init and start the Data Sender" ); return false; } - mCollectionInspectionWorkerThread->subscribeToDataReadyToPublish( + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda + mCollectedDataReadyToPublish->subscribeToNewDataAvailable( std::bind( &DataSenderManagerWorkerThread::onDataReadyToPublish, mDataSenderManagerWorkerThread.get() ) ); /*************************DataSender bootstrap end*********************************/ @@ -509,41 +585,35 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) // CollectionScheme Ingestion module executes in the context for the offboardconnectivity thread. Upcoming // messages are expected to come either on the decoder manifest topic or the collectionScheme topic or both // ( eventually ). - mSchemaPtr = std::make_shared( mConnectivityChannelReceiveDecoderManifest, - mConnectivityChannelReceiveCollectionSchemeList, - mConnectivityChannelSendCheckin ); + mSchemaPtr = + std::make_shared( mReceiverDecoderManifest, mReceiverCollectionSchemeList, mSenderCheckin ); /*****************************CollectionScheme Management bootstrap begin*****************************/ - // Create and connect the CollectionScheme Manager - mCollectionSchemeManagerPtr = std::make_shared(); + // Allow CollectionSchemeManagement to send checkins through the Schema Object Callback + mCheckinSender = std::make_shared( + mSchemaPtr, + config["staticConfig"]["publishToCloudParameters"]["collectionSchemeManagementCheckinIntervalMs"] + .asU32Required() ); - if ( !mCollectionSchemeManagerPtr->init( - config["staticConfig"]["publishToCloudParameters"]["collectionSchemeManagementCheckinIntervalMs"] - .asU32Required(), - mPersistDecoderManifestCollectionSchemesAndData, - mCANIDTranslator -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - rawDataBufferManager -#endif - ) ) - { - FWE_LOG_ERROR( "Failed to init the CollectionScheme Manager" ); - return false; - } + // Create and connect the CollectionScheme Manager + mCollectionSchemeManagerPtr = std::make_shared( + mPersistDecoderManifestCollectionSchemesAndData, mCANIDTranslator, mCheckinSender, mRawBufferManager ); // Make sure the CollectionScheme Ingestion can notify the CollectionScheme Manager about the arrival - // of new artifacts over the offboardconnectivity channel. + // of new artifacts over the offboardconnectivity receiver. + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mSchemaPtr->subscribeToCollectionSchemeUpdate( std::bind( &CollectionSchemeManager::onCollectionSchemeUpdate, mCollectionSchemeManagerPtr.get(), std::placeholders::_1 ) ); + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mSchemaPtr->subscribeToDecoderManifestUpdate( std::bind( &CollectionSchemeManager::onDecoderManifestUpdate, mCollectionSchemeManagerPtr.get(), std::placeholders::_1 ) ); // Make sure the CollectionScheme Manager can notify the Inspection Engine about the availability of // a new set of collection CollectionSchemes. + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToInspectionMatrixChange( std::bind( &CollectionInspectionWorkerThread::onChangeInspectionMatrix, mCollectionInspectionWorkerThread.get(), @@ -552,24 +622,30 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) #ifdef FWE_FEATURE_VISION_SYSTEM_DATA // Make sure the CollectionScheme Manager can notify the Data Sender about the availability of // a new set of collection CollectionSchemes. - mCollectionSchemeManagerPtr->subscribeToCollectionSchemeListChange( - std::bind( &DataSenderManagerWorkerThread::onChangeCollectionSchemeList, - mDataSenderManagerWorkerThread.get(), - std::placeholders::_1 ) ); - mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( - std::bind( &DataSenderIonWriter::onChangeOfActiveDictionary, - ionWriter.get(), - std::placeholders::_1, - std::placeholders::_2 ) ); -#endif + if ( visionSystemDataSender != nullptr ) + { + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda + mCollectionSchemeManagerPtr->subscribeToCollectionSchemeListChange( + std::bind( &VisionSystemDataSender::onChangeCollectionSchemeList, + visionSystemDataSender.get(), + std::placeholders::_1 ) ); + } - // Allow CollectionSchemeManagement to send checkins through the Schema Object Callback - mCollectionSchemeManagerPtr->setSchemaListenerPtr( mSchemaPtr ); + if ( ionWriter != nullptr ) + { + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda + mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( + std::bind( &DataSenderIonWriter::onChangeOfActiveDictionary, + ionWriter.get(), + std::placeholders::_1, + std::placeholders::_2 ) ); + } +#endif /********************************Data source bootstrap start*******************************/ auto obdOverCANModuleInit = false; - mCANDataConsumer = std::make_unique( signalBufferPtr ); + mCANDataConsumer = std::make_unique( signalBufferDistributor ); for ( unsigned i = 0; i < config["networkInterfaces"].getArraySizeRequired(); i++ ) { const auto networkInterfaceConfig = config["networkInterfaces"][i]; @@ -603,6 +679,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) FWE_LOG_ERROR( "Failed to initialize CANDataSource" ); return false; } + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &CANDataSource::onChangeOfActiveDictionary, canSourcePtr.get(), @@ -618,7 +695,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) obdOverCANModuleInit = true; auto obdConfig = networkInterfaceConfig[OBD_INTERFACE_TYPE]; if ( obdOverCANModule->init( - signalBufferPtr, + signalBufferDistributor, obdConfig["interfaceName"].asStringRequired(), obdConfig["pidRequestIntervalSeconds"].asU32Required(), obdConfig["dtcRequestIntervalSeconds"].asU32Required(), @@ -633,11 +710,13 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) return false; } + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &OBDOverCANModule::onChangeOfActiveDictionary, mOBDOverCANModule.get(), std::placeholders::_1, std::placeholders::_2 ) ); + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToInspectionMatrixChange( std::bind( &OBDOverCANModule::onChangeInspectionMatrix, mOBDOverCANModule.get(), @@ -651,15 +730,17 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } else if ( interfaceType == EXTERNAL_CAN_INTERFACE_TYPE ) { - if ( mExternalCANDataSource == nullptr ) + if ( mExternalCANDataSource != nullptr ) { - mExternalCANDataSource = std::make_unique( *mCANDataConsumer ); - mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( - std::bind( &ExternalCANDataSource::onChangeOfActiveDictionary, - mExternalCANDataSource.get(), - std::placeholders::_1, - std::placeholders::_2 ) ); + continue; } + mExternalCANDataSource = std::make_unique( *mCANDataConsumer ); + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda + mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( + std::bind( &ExternalCANDataSource::onChangeOfActiveDictionary, + mExternalCANDataSource.get(), + std::placeholders::_1, + std::placeholders::_2 ) ); } #ifdef FWE_FEATURE_ROS2 else if ( interfaceType == ROS2_INTERFACE_TYPE ) @@ -669,8 +750,10 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) { return false; } - mROS2DataSource = std::make_shared( ros2Config, signalBufferPtr, rawDataBufferManager ); + mROS2DataSource = + std::make_shared( ros2Config, signalBufferDistributor, mRawBufferManager ); mROS2DataSource->connect(); + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &ROS2DataSource::onChangeOfActiveDictionary, mROS2DataSource.get(), @@ -686,6 +769,24 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) /********************************Data source bootstrap end*******************************/ + // For asynchronous connect the call needs to be done after all senders and receivers are + // created and all receiver listeners are subscribed. + if ( !mConnectivityModule->connect() ) + { + return false; + } + + if ( ( mRemoteProfiler != nullptr ) && ( !mRemoteProfiler->start() ) ) + { + FWE_LOG_WARN( "Failed to start the Remote Profiler - No remote profiling available until FWE restart" ); + } + + if ( !mCheckinSender->start() ) + { + FWE_LOG_ERROR( "Failed to start the Checkin thread" ); + return false; + } + // Only start the CollectionSchemeManager after all listeners have subscribed, otherwise // they will not be notified of the initial decoder manifest and collection schemes that are // read from persistent memory: @@ -698,7 +799,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) #ifdef FWE_FEATURE_IWAVE_GPS /********************************IWave GPS Example NMEA reader *********************************/ - mIWaveGpsSource = std::make_shared( signalBufferPtr ); + mIWaveGpsSource = std::make_shared( signalBufferDistributor ); std::string pathToNmeaSource; CANChannelNumericID canChannel{}; CANRawFrameID canRawFrameId{}; @@ -734,6 +835,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) FWE_LOG_ERROR( "IWaveGps initialization failed" ); return false; } + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &IWaveGpsSource::onChangeOfActiveDictionary, mIWaveGpsSource.get(), @@ -746,7 +848,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) #ifdef FWE_FEATURE_EXTERNAL_GPS /********************************External GPS Example NMEA reader *********************************/ - mExternalGpsSource = std::make_shared( signalBufferPtr ); + mExternalGpsSource = std::make_shared( signalBufferDistributor ); bool externalGpsInitSuccessful = false; if ( config["staticConfig"].isMember( CONFIG_SECTION_EXTERNAL_GPS ) ) { @@ -769,6 +871,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } if ( externalGpsInitSuccessful ) { + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &ExternalGpsSource::onChangeOfActiveDictionary, mExternalGpsSource.get(), @@ -786,7 +889,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) #ifdef FWE_FEATURE_AAOS_VHAL /********************************AAOS VHAL Example reader *********************************/ - mAaosVhalSource = std::make_shared( signalBufferPtr ); + mAaosVhalSource = std::make_shared( signalBufferDistributor ); bool aaosVhalInitSuccessful = false; if ( config["staticConfig"].isMember( CONFIG_SECTION_AAOS_VHAL ) ) { @@ -805,6 +908,7 @@ IoTFleetWiseEngine::connect( const Json::Value &jsonConfig ) } if ( aaosVhalInitSuccessful ) { + // coverity[autosar_cpp14_a18_9_1_violation] std::bind is easier to maintain than extra lambda mCollectionSchemeManagerPtr->subscribeToActiveDecoderDictionaryChange( std::bind( &AaosVhalSource::onChangeOfActiveDictionary, mAaosVhalSource.get(), @@ -891,6 +995,12 @@ IoTFleetWiseEngine::disconnect() return false; } + if ( !mCheckinSender->stop() ) + { + FWE_LOG_ERROR( "Failed to stop the Checkin thread" ); + return false; + } + for ( auto &source : mCANDataSources ) { if ( !source->disconnect() ) @@ -900,7 +1010,7 @@ IoTFleetWiseEngine::disconnect() } } - if ( mConnectivityModule->isAlive() && ( !mConnectivityModule->disconnect() ) ) + if ( !mConnectivityModule->disconnect() ) { FWE_LOG_ERROR( "Could not disconnect the offboard connectivity" ); return false; @@ -1082,8 +1192,9 @@ IoTFleetWiseEngine::getVehiclePropertyInfo() } return propertyInfo; } + void -IoTFleetWiseEngine::setVehicleProperty( uint32_t signalId, double value ) +IoTFleetWiseEngine::setVehicleProperty( uint32_t signalId, const DecodedSignalValue &value ) { if ( mAaosVhalSource == nullptr ) { @@ -1096,8 +1207,8 @@ IoTFleetWiseEngine::setVehicleProperty( uint32_t signalId, double value ) std::string IoTFleetWiseEngine::getStatusSummary() { - if ( mConnectivityModule == nullptr || mCollectionSchemeManagerPtr == nullptr || - mConnectivityChannelSendVehicleData == nullptr || mOBDOverCANModule == nullptr ) + if ( mConnectivityModule == nullptr || mCollectionSchemeManagerPtr == nullptr || mSenderVehicleData == nullptr || + mOBDOverCANModule == nullptr ) { return ""; } @@ -1120,7 +1231,7 @@ IoTFleetWiseEngine::getStatusSummary() } status += "\n"; - status += "Payloads sent: " + std::to_string( mConnectivityChannelSendVehicleData->getPayloadCountSent() ) + "\n\n"; + status += "Payloads sent: " + std::to_string( mSenderVehicleData->getPayloadCountSent() ) + "\n\n"; return status; } diff --git a/src/IoTFleetWiseEngine.h b/src/IoTFleetWiseEngine.h index 13c0b08e..038564b5 100644 --- a/src/IoTFleetWiseEngine.h +++ b/src/IoTFleetWiseEngine.h @@ -7,19 +7,23 @@ #include "CANDataSource.h" #include "CANInterfaceIDTranslator.h" #include "CacheAndPersist.h" +#include "CheckinSender.h" #include "Clock.h" #include "ClockHandler.h" -#include "CollectionInspectionAPITypes.h" +#include "CollectionInspectionEngine.h" #include "CollectionInspectionWorkerThread.h" #include "CollectionSchemeManager.h" #include "DataSenderManager.h" #include "DataSenderManagerWorkerThread.h" +#include "DataSenderTypes.h" #include "ExternalCANDataSource.h" -#include "IConnectivityChannel.h" #include "IConnectivityModule.h" +#include "IReceiver.h" +#include "ISender.h" #include "OBDDataTypes.h" #include "OBDOverCANModule.h" #include "PayloadManager.h" +#include "RawDataManager.h" #include "RemoteProfiler.h" #include "Schema.h" #include "Signal.h" @@ -28,6 +32,7 @@ #include "TimeTypes.h" #include "Timer.h" #include +#include #include #include #include @@ -45,11 +50,20 @@ #ifdef FWE_FEATURE_IWAVE_GPS #include "IWaveGpsSource.h" #endif -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "S3Sender.h" +#ifdef FWE_FEATURE_S3 +#include #include +#if ( AWS_SDK_VERSION_MAJOR > 1 ) || \ + ( ( AWS_SDK_VERSION_MAJOR == 1 ) && \ + ( ( AWS_SDK_VERSION_MINOR > 11 ) || ( ( AWS_SDK_VERSION_MINOR == 11 ) && ( AWS_SDK_VERSION_PATCH >= 224 ) ) ) ) +#include +#else #include #endif +#endif +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +#include "S3Sender.h" +#endif #ifdef FWE_FEATURE_ROS2 #include "ROS2DataSource.h" #endif @@ -78,7 +92,7 @@ class IoTFleetWiseEngine IoTFleetWiseEngine( IoTFleetWiseEngine && ) = delete; IoTFleetWiseEngine &operator=( IoTFleetWiseEngine && ) = delete; - bool connect( const Json::Value &jsonConfig ); + bool connect( const Json::Value &jsonConfig, const boost::filesystem::path &configFileDirectoryPath ); bool start(); bool stop(); bool disconnect(); @@ -132,7 +146,7 @@ class IoTFleetWiseEngine * @param signalId Signal ID * @param value Vehicle property value */ - void setVehicleProperty( uint32_t signalId, double value ); + void setVehicleProperty( uint32_t signalId, const DecodedSignalValue &value ); #endif /** @@ -149,7 +163,7 @@ class IoTFleetWiseEngine static void doWork( void *data ); public: - std::shared_ptr mCollectedDataReadyToPublish; + std::shared_ptr mCollectedDataReadyToPublish; // Object for handling persistency for decoder manifest, collectionSchemes and edge to cloud payload std::shared_ptr mPersistDecoderManifestCollectionSchemesAndData; @@ -173,26 +187,33 @@ class IoTFleetWiseEngine std::unique_ptr mCANDataConsumer; std::shared_ptr mConnectivityModule; - std::shared_ptr mConnectivityChannelSendVehicleData; - std::shared_ptr mConnectivityChannelSendCheckin; - std::shared_ptr mConnectivityChannelReceiveCollectionSchemeList; - std::shared_ptr mConnectivityChannelReceiveDecoderManifest; + std::shared_ptr mSenderVehicleData; + std::shared_ptr mSenderCheckin; + std::shared_ptr mReceiverCollectionSchemeList; + std::shared_ptr mReceiverDecoderManifest; std::shared_ptr mPayloadManager; std::shared_ptr mSchemaPtr; + std::shared_ptr mCheckinSender; std::shared_ptr mCollectionSchemeManagerPtr; + std::shared_ptr mRawBufferManager; + std::shared_ptr mCollectionInspectionEngine; std::shared_ptr mCollectionInspectionWorkerThread; std::shared_ptr mDataSenderManager; std::shared_ptr mDataSenderManagerWorkerThread; std::unique_ptr mRemoteProfiler; - std::shared_ptr mConnectivityChannelMetricsUpload; - std::shared_ptr mConnectivityChannelLogsUpload; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::shared_ptr mSenderMetrics; + std::shared_ptr mSenderLogs; +#ifdef FWE_FEATURE_S3 std::shared_ptr mAwsCredentialsProvider; std::shared_ptr mTransferManagerExecutor; + std::mutex mTransferManagerExecutorMutex; + std::shared_ptr getTransferManagerExecutor(); +#endif +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA std::shared_ptr mS3Sender; #endif #ifdef FWE_FEATURE_IWAVE_GPS diff --git a/src/Listener.h b/src/Listener.h index a00df10d..bc21fa9a 100644 --- a/src/Listener.h +++ b/src/Listener.h @@ -25,7 +25,7 @@ class ThreadSafeListeners ThreadSafeListeners() = default; virtual ~ThreadSafeListeners() { - MutexLock lock( mMutex ); + std::lock_guard lock( mMutex ); mContainer.clear(); } ThreadSafeListeners( const ThreadSafeListeners & ) = delete; @@ -40,7 +40,7 @@ class ThreadSafeListeners void subscribe( T callback ) { - MutexLock lock( mMutex ); + std::lock_guard lock( mMutex ); CallbackContainer &container = getContainer(); container.emplace_back( callback ); @@ -54,7 +54,7 @@ class ThreadSafeListeners void notify( Args... args ) const { - MutexLock lock( mMutex ); + std::lock_guard lock( mMutex ); ContainerInvocationState state( this ); @@ -67,8 +67,6 @@ class ThreadSafeListeners private: // Container to store the list of listeners to this thread using CallbackContainer = std::vector; - // Mutex to protect the storage from concurrent reads and writes - using MutexLock = std::lock_guard; // Container for all listeners registered. mutable CallbackContainer mContainer; // Temporary container used during modification via subscribe/unsubscribe diff --git a/src/OBDDataDecoder.cpp b/src/OBDDataDecoder.cpp index eece7b52..cc43f6fd 100644 --- a/src/OBDDataDecoder.cpp +++ b/src/OBDDataDecoder.cpp @@ -385,7 +385,7 @@ OBDDataDecoder::calculateValueFromFormula( PID pid, static_cast( static_cast( rawData ) * formula.mFactor + formula.mOffset ); TraceModule::get().incrementVariable( TraceVariable::OBD_POSSIBLE_PRECISION_LOSS_UINT64 ); } - info.mPIDsToValues.emplace( formula.mSignalID, OBDSignal( calculatedValue, signalType ) ); + info.mPIDsToValues.emplace( formula.mSignalID, DecodedSignalValue( calculatedValue, signalType ) ); break; } case ( SignalType::INT64 ): { @@ -401,7 +401,7 @@ OBDDataDecoder::calculateValueFromFormula( PID pid, static_cast( static_cast( rawData ) * formula.mFactor + formula.mOffset ); TraceModule::get().incrementVariable( TraceVariable::OBD_POSSIBLE_PRECISION_LOSS_INT64 ); } - info.mPIDsToValues.emplace( formula.mSignalID, OBDSignal( calculatedValue, signalType ) ); + info.mPIDsToValues.emplace( formula.mSignalID, DecodedSignalValue( calculatedValue, signalType ) ); break; } // For any other type, we can safely cast everything to double as only int64 and uint64 can't @@ -409,7 +409,7 @@ OBDDataDecoder::calculateValueFromFormula( PID pid, default: { double calculatedValue = static_cast( static_cast( rawData ) ) * formula.mFactor + formula.mOffset; - info.mPIDsToValues.emplace( formula.mSignalID, OBDSignal( calculatedValue, signalType ) ); + info.mPIDsToValues.emplace( formula.mSignalID, DecodedSignalValue( calculatedValue, signalType ) ); } } } diff --git a/src/OBDDataTypes.h b/src/OBDDataTypes.h index 31e417ba..bb843f57 100644 --- a/src/OBDDataTypes.h +++ b/src/OBDDataTypes.h @@ -4,6 +4,7 @@ #pragma once #include "EnumUtility.h" +#include "SignalTypes.h" #include "TimeTypes.h" #include "VehicleDataSourceTypes.h" #include @@ -20,35 +21,6 @@ namespace Aws namespace IoTFleetWise { -union OBDValue { - double doubleVal; - uint64_t uint64Val; - int64_t int64Val; -}; - -struct OBDSignal -{ - OBDValue signalValue; - SignalType signalType; - - template - OBDSignal( T val, SignalType type ) - : signalType( type ) - { - switch ( signalType ) - { - case SignalType::UINT64: - signalValue.uint64Val = static_cast( val ); - break; - case SignalType::INT64: - signalValue.int64Val = static_cast( val ); - break; - default: - signalValue.doubleVal = static_cast( val ); - } - } -}; - // List of OBD Service IDs/ Modes enum class SID : uint32_t { @@ -229,7 +201,7 @@ struct DTCInfo struct EmissionInfo { SID mSID; - std::map mPIDsToValues; + std::map mPIDsToValues; }; // Structure of a single PID OBD request. diff --git a/src/OBDOverCANECU.cpp b/src/OBDOverCANECU.cpp index e8fbb4e3..4b5453a0 100644 --- a/src/OBDOverCANECU.cpp +++ b/src/OBDOverCANECU.cpp @@ -2,9 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 #include "OBDOverCANECU.h" +#include "CollectionInspectionAPITypes.h" #include "EnumUtility.h" #include "ISOTPOverCANOptions.h" #include "LoggingModule.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "TraceModule.h" #include @@ -29,7 +31,7 @@ OBDOverCANECU::init( const std::string &gatewayCanInterfaceName, const uint32_t rxId, const uint32_t txId, bool isExtendedId, - SignalBufferPtr &signalBufferPtr, + SignalBufferDistributorPtr &signalBufferDistributor, int broadcastSocket ) { ISOTPOverCANSenderReceiverOptions optionsECU; @@ -39,7 +41,7 @@ OBDOverCANECU::init( const std::string &gatewayCanInterfaceName, optionsECU.mIsExtendedId = isExtendedId; optionsECU.mBroadcastSocket = broadcastSocket; mOBDDataDecoder = obdDataDecoder; - mSignalBufferPtr = signalBufferPtr; + mSignalBufferDistributor = signalBufferDistributor; std::stringstream streamRx; streamRx << std::hex << rxId; @@ -130,7 +132,7 @@ OBDOverCANECU::requestReceiveEmissionPIDs( const SID sid ) requestReceivePIDs( pidItr, sid, pids, info ); } - pushPIDs( info, mClock->systemTimeSinceEpochMs(), mSignalBufferPtr, mStreamRxID ); + pushPIDs( info, mClock->systemTimeSinceEpochMs(), mSignalBufferDistributor, mStreamRxID ); } return numRequests; } @@ -138,7 +140,7 @@ OBDOverCANECU::requestReceiveEmissionPIDs( const SID sid ) void OBDOverCANECU::pushPIDs( const EmissionInfo &info, Timestamp receptionTime, - SignalBufferPtr &signalBufferPtr, + SignalBufferDistributorPtr &signalBufferDistributor, const std::string &streamRxID ) { CollectedSignalsGroup collectedSignalsGroup; @@ -173,17 +175,7 @@ OBDOverCANECU::pushPIDs( const EmissionInfo &info, } } - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().addToAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignalsGroup.size() ); - - if ( !signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().subtractFromAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignalsGroup.size() ); - FWE_LOG_WARN( "Signal buffer full with ECU " + streamRxID ); - } + signalBufferDistributor->push( CollectedDataFrame( collectedSignalsGroup ) ); } bool diff --git a/src/OBDOverCANECU.h b/src/OBDOverCANECU.h index 1612f456..db1e5399 100644 --- a/src/OBDOverCANECU.h +++ b/src/OBDOverCANECU.h @@ -44,7 +44,7 @@ class OBDOverCANECU * @param rxId CAN Receive ID * @param txId CAN Transmit ID * @param isExtendedId CAN ID is standard(11-bit) or extended(29-bit) - * @param signalBufferPtr Signal Buffer shared pointer + * @param signalBufferDistributor Signal buffer distributor * @param broadcastSocket Sending broadcast requests by using broadcast socket * @return True if initialization of ISO-TP is successful */ @@ -53,7 +53,7 @@ class OBDOverCANECU const uint32_t rxId, const uint32_t txId, bool isExtendedId, - SignalBufferPtr &signalBufferPtr, + SignalBufferDistributorPtr &signalBufferDistributor, int broadcastSocket ); /** @@ -84,12 +84,12 @@ class OBDOverCANECU * * @param info PID vaues to push * @param receptionTime Timestamp of reception - * @param signalBufferPtr Signal Buffer shared pointer + * @param signalBufferDistributor Signal buffer distributor * @param streamRxID CAN Receive ID */ static void pushPIDs( const EmissionInfo &info, Timestamp receptionTime, - SignalBufferPtr &signalBufferPtr, + SignalBufferDistributorPtr &signalBufferDistributor, const std::string &streamRxID ); /** @@ -166,8 +166,7 @@ class OBDOverCANECU std::vector mTxPDU; std::vector mRxPDU; std::shared_ptr mOBDDataDecoder; - // Signal Buffer shared pointer. This is a multiple producer single consumer queue - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; // The PIDs to request from ECU. This assignment would come from OBDOverCANModule std::unordered_map> mPIDsToRequest; // can iso-tp tranceiver diff --git a/src/OBDOverCANModule.cpp b/src/OBDOverCANModule.cpp index 8e6aa8bc..65f24ecf 100644 --- a/src/OBDOverCANModule.cpp +++ b/src/OBDOverCANModule.cpp @@ -6,8 +6,8 @@ #include "ISOTPOverCANOptions.h" #include "LoggingModule.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" -#include "TraceModule.h" #include #include #include @@ -44,19 +44,19 @@ OBDOverCANModule::~OBDOverCANModule() } bool -OBDOverCANModule::init( SignalBufferPtr signalBufferPtr, +OBDOverCANModule::init( SignalBufferDistributorPtr signalBufferDistributor, const std::string &gatewayCanInterfaceName, uint32_t pidRequestIntervalSeconds, uint32_t dtcRequestIntervalSeconds, bool broadcastRequests ) { - if ( ( signalBufferPtr.get() == nullptr ) ) + if ( ( signalBufferDistributor.get() == nullptr ) ) { FWE_LOG_ERROR( "Received Buffer nullptr" ); return false; } - mSignalBufferPtr = signalBufferPtr; + mSignalBufferDistributor = signalBufferDistributor; mOBDDataDecoder = std::make_shared( mDecoderDictionaryPtr ); mGatewayCanInterfaceName = gatewayCanInterfaceName; mPIDRequestIntervalSeconds = pidRequestIntervalSeconds; @@ -235,20 +235,8 @@ OBDOverCANModule::doWork( void *data ) // there was a OBD request that did not return any SID::STORED_DTCs if ( successfulDTCRequest ) { - TraceModule::get().incrementAtomicVariable( - TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().incrementAtomicVariable( - TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DTCS ); - - if ( !obdModule->mSignalBufferPtr->push( - CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ) - { - TraceModule::get().decrementAtomicVariable( - TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().decrementAtomicVariable( - TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DTCS ); - FWE_LOG_WARN( "DTC Buffer full" ); - } + obdModule->mSignalBufferDistributor->push( + CollectedDataFrame( std::make_shared( dtcInfo ) ) ); } } } @@ -495,7 +483,7 @@ OBDOverCANModule::initECUs( bool isExtendedID, std::vector &canIDRespo rxID, getTxIDByRxID( isExtendedID, rxID ), isExtendedID, - mSignalBufferPtr, + mSignalBufferDistributor, broadcastSocket ) ) { return false; @@ -597,7 +585,7 @@ OBDOverCANModule::setExternalPIDResponse( PID pid, std::vector response { return; } - OBDOverCANECU::pushPIDs( info, mClock->systemTimeSinceEpochMs(), mSignalBufferPtr, "" ); + OBDOverCANECU::pushPIDs( info, mClock->systemTimeSinceEpochMs(), mSignalBufferDistributor, "" ); } void diff --git a/src/OBDOverCANModule.h b/src/OBDOverCANModule.h index 9d4fa0fe..d71a37a2 100644 --- a/src/OBDOverCANModule.h +++ b/src/OBDOverCANModule.h @@ -58,7 +58,7 @@ class OBDOverCANModule /** * @brief Initializes the OBD Diagnostic Session. - * @param signalBufferPtr Signal Buffer shared pointer. + * @param signalBufferDistributor Signal buffer distributor * @param gatewayCanInterfaceName CAN IF Name where the OBD stack on the ECU * is running. Typically on the Gateway ECU. * @param pidRequestIntervalSeconds Interval in seconds used to schedule PID requests @@ -67,7 +67,7 @@ class OBDOverCANModule * @return True if successful. False if both pidRequestIntervalSeconds * and dtcRequestIntervalSeconds are zero i.e. no collection */ - bool init( SignalBufferPtr signalBufferPtr, + bool init( SignalBufferDistributorPtr signalBufferDistributor, const std::string &gatewayCanInterfaceName, uint32_t pidRequestIntervalSeconds, uint32_t dtcRequestIntervalSeconds, @@ -99,17 +99,6 @@ class OBDOverCANModule // We need this to know whether DTCs should be requested or not void onChangeInspectionMatrix( const std::shared_ptr &inspectionMatrix ); - /** - * @brief Handle of the Signal Output Buffer. This buffer shared between Collection Engine - * Vehicle Data Consumer and OBDOverCANModule - * @return shared object pointer to the Signal buffer. - */ - inline SignalBufferPtr - getSignalBufferPtr() const - { - return mSignalBufferPtr; - } - /** * @brief Gets a list of PIDs to request externally * @return List of PIDs @@ -191,8 +180,7 @@ class OBDOverCANModule // Decoder Manifest and campaigns availability Signal Signal mDataAvailableWait; - // Signal Buffer shared pointer - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; uint32_t mPIDRequestIntervalSeconds{ 0 }; uint32_t mDTCRequestIntervalSeconds{ 0 }; bool mBroadcastRequests{ false }; diff --git a/src/PayloadManager.cpp b/src/PayloadManager.cpp index 5f28b7ec..c1cb6c29 100644 --- a/src/PayloadManager.cpp +++ b/src/PayloadManager.cpp @@ -21,12 +21,8 @@ PayloadManager::PayloadManager( std::shared_ptr persistencyPtr bool PayloadManager::storeData( const std::uint8_t *buf, size_t size, - const CollectionSchemeParams &collectionSchemeParams -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - const S3UploadParams &s3UploadParams -#endif -) + const Json::Value &metadata, + const std::string &filename ) { if ( mPersistencyPtr == nullptr ) { @@ -41,19 +37,6 @@ PayloadManager::storeData( const std::uint8_t *buf, return false; } - std::string filename; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - if ( s3UploadParams != S3UploadParams() ) - { - filename = s3UploadParams.objectName; - } - else -#endif - { - filename = std::to_string( collectionSchemeParams.eventID ) + "-" + - std::to_string( collectionSchemeParams.triggerTime ) + ".bin"; - } - ErrorCode writeStatus = mPersistencyPtr->write( buf, size, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); if ( writeStatus != ErrorCode::SUCCESS ) { @@ -69,66 +52,44 @@ PayloadManager::storeData( const std::uint8_t *buf, FWE_LOG_TRACE( "Payload of size : " + std::to_string( size ) + " Bytes has been successfully persisted in file " + filename ); - storeMetadata( filename, - size, - collectionSchemeParams -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - s3UploadParams -#endif - ); + mPersistencyPtr->addMetadata( metadata ); + FWE_LOG_TRACE( "Metadata for file " + filename + " has been successfully added" ); + return true; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA bool -PayloadManager::storeIonData( std::unique_ptr streambuf, std::string filename ) +PayloadManager::storeData( std::streambuf &streambuf, const Json::Value &metadata, const std::string &filename ) { - if ( streambuf == nullptr ) + // TODO: We don't fully support persisting a stream payload yet, so only the stream content will be saved + static_cast( metadata ); + + if ( mPersistencyPtr == nullptr ) { - FWE_LOG_ERROR( "No stream provided" ); + FWE_LOG_ERROR( "No CacheAndPersist module provided" ); return false; } - ErrorCode writeStatus = mPersistencyPtr->write( std::move( streambuf ), DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); + ErrorCode writeStatus = mPersistencyPtr->write( streambuf, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); if ( writeStatus != ErrorCode::SUCCESS ) { FWE_LOG_ERROR( "Failed to persist collected data on disk" ); + TraceModule::get().incrementVariable( TraceVariable::PM_STORE_ERROR ); + if ( writeStatus == ErrorCode::MEMORY_FULL ) + { + TraceModule::get().incrementVariable( TraceVariable::PM_MEMORY_INSUFFICIENT ); + } return false; } FWE_LOG_TRACE( "Payload has been successfully persisted in file " + filename ); return true; } -#endif void -PayloadManager::storeMetadata( const std::string filename, - size_t size, - const CollectionSchemeParams &collectionSchemeParams -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - const S3UploadParams &s3UploadParams -#endif -) +PayloadManager::storeMetadata( const Json::Value &metadata ) { - std::lock_guard lock( mMetadataMutex ); - Json::Value metadata; - metadata["filename"] = filename; - metadata["payloadSize"] = static_cast( size ); - metadata["compressionRequired"] = collectionSchemeParams.compression; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - if ( s3UploadParams != S3UploadParams() ) - { - metadata["s3UploadMetadata"]["bucketName"] = s3UploadParams.bucketName; - metadata["s3UploadMetadata"]["bucketOwner"] = s3UploadParams.bucketOwner; - metadata["s3UploadMetadata"]["region"] = s3UploadParams.region; - metadata["s3UploadMetadata"]["uploadID"] = s3UploadParams.uploadID; - metadata["s3UploadMetadata"]["partNumber"] = s3UploadParams.multipartID; - } -#endif mPersistencyPtr->addMetadata( metadata ); - FWE_LOG_TRACE( "Metadata for file " + filename + " has been successfully added" ); } ErrorCode @@ -174,5 +135,27 @@ PayloadManager::retrievePayload( uint8_t *buf, size_t size, const std::string &f return ErrorCode::SUCCESS; } +ErrorCode +PayloadManager::retrievePayloadLazily( std::ifstream &fileStream, const std::string &filename ) +{ + return mPersistencyPtr->read( fileStream, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); +} + +void +PayloadManager::deletePayload( const std::string &filename ) +{ + if ( mPersistencyPtr == nullptr ) + { + FWE_LOG_ERROR( "No CacheAndPersist module provided" ); + return; + } + + auto status = mPersistencyPtr->erase( DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); + if ( status != ErrorCode::SUCCESS ) + { + FWE_LOG_ERROR( "Failed to delete persisted file " + filename ); + } +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/src/PayloadManager.h b/src/PayloadManager.h index 14e852bf..5c39b001 100644 --- a/src/PayloadManager.h +++ b/src/PayloadManager.h @@ -4,54 +4,18 @@ #pragma once #include "CacheAndPersist.h" -#include "ISender.h" #include #include #include #include -#include -#include - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA #include -#endif +#include namespace Aws { namespace IoTFleetWise { -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -/** - * @brief Struct that specifies the persistence and transmission attributes - * for the S3 upload - */ -struct S3UploadParams -{ - std::string region; // bucket region, set on the campaign level, attribute of S3 client - std::string bucketName; // bucket name, set on the campaign level, attribute of S3 request - std::string bucketOwner; // bucket owner account ID, set on the campaign level, attribute of S3 request - std::string objectName; // object key, attribute of S3 request - std::string uploadID; // upload ID of the multipart upload - uint16_t multipartID{ 0 }; // multipartID of a single part of the multipart upload, allowed values are 1 to 10000 - -public: - bool - operator==( const S3UploadParams &other ) const - { - return ( bucketName == other.bucketName ) && ( bucketOwner == other.bucketOwner ) && - ( objectName == other.objectName ) && ( region == other.region ) && ( uploadID == other.uploadID ) && - ( multipartID == other.multipartID ); - } - - bool - operator!=( const S3UploadParams &other ) const - { - return !( *this == other ); - } -}; -#endif - /** * @brief Class that handles offline data storage/retrieval and data compression before transmission */ @@ -63,53 +27,38 @@ class PayloadManager virtual ~PayloadManager() = default; /** - * @brief Prepares and writes the payload data to storage. Constructs and writes payload metadata JSON object. + * @brief Write the payload data to storage together with metadata. + * + * @param buf buffer containing payload + * @param size number of accessible bytes in buf + * @param metadata metadata associated with the payload + * @param filename full name of the file to store * * @return true if data was persisted and metadata was added, else false */ - virtual bool storeData( - const std::uint8_t *buf, /**< buffer containing payload */ - size_t size, /**< size number of accessible bytes in buf */ - const CollectionSchemeParams &collectionSchemeParams /**< object containing collectionScheme related - metadata for data persistency and transmission */ -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - const struct S3UploadParams &s3UploadParams = - S3UploadParams() /**< object containing metadata related to the S3 upload for data persistency and - transmission. If object is not passed or contains default values, no S3 related - parameters are written to the JSON metadata file. */ -#endif - ); + virtual bool storeData( const std::uint8_t *buf, + size_t size, + const Json::Value &metadata, + const std::string &filename ); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA /** - * @brief Calls CacheAndPersist module to write the Ion file + * @brief Calls CacheAndPersist module to write the given stream to file * - * @param streambuf stream with the Ion data + * @param streambuf stream with data to be persisted + * @param metadata metadata associated with the payload * @param filename full name of the file to store * * @return true if data was persisted and metadata was added, else false */ - virtual bool storeIonData( std::unique_ptr streambuf, std::string filename ); -#endif + virtual bool storeData( std::streambuf &streambuf, const Json::Value &metadata, const std::string &filename ); /** - * @brief Constructs and writes payload metadata JSON object. + * @brief Store only metadata for a file. This can be used when sending a file failed and we want to keep persisting + * it. + * + * @param metadata metadata associated with the payload */ - virtual void storeMetadata( - const std::string filename, /**< filename file to construct metadata for */ - size_t size, /**< size of the payload */ - const CollectionSchemeParams - &collectionSchemeParams /**< collectionSchemeParams object containing collectionScheme - related metadata for data persistency and transmission */ -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - const struct S3UploadParams &s3UploadParams = - S3UploadParams() /**< object containing metadata related to the S3 upload for data persistency and - transmission. If object is not passed or contains default values, no S3 related - parameters are written to the JSON metadata file. */ -#endif - ); + virtual void storeMetadata( const Json::Value &metadata ); /** * @brief Retrieves metadata for all persisted files from the JSON file and removes extracted metadata from the @@ -132,9 +81,27 @@ class PayloadManager */ virtual ErrorCode retrievePayload( uint8_t *buf, size_t size, const std::string &filename ); + /** + * @brief Retrieves persisted payload from the file as a stream to be read on demand. + * + * @param fileStream the file stream that will point to the file being requested + * @param filename filename to retrieve payload + * + * @return SUCCESS if metadata was successfully retrieved, FILESYSTEM_ERROR for other errors + */ + virtual ErrorCode retrievePayloadLazily( std::ifstream &fileStream, const std::string &filename ); + + /** + * @brief Delete a payload file. Only a the payload file is deleted, not the medadata. + * + * This normally should be called after retrievePayloadLazily. + * + * @param filename + */ + virtual void deletePayload( const std::string &filename ); + private: std::shared_ptr mPersistencyPtr; - std::mutex mMetadataMutex; }; } // namespace IoTFleetWise diff --git a/src/CheckinAndPersistency.cpp b/src/Persistency.cpp similarity index 68% rename from src/CheckinAndPersistency.cpp rename to src/Persistency.cpp index 34fadfde..08e28738 100644 --- a/src/CheckinAndPersistency.cpp +++ b/src/Persistency.cpp @@ -7,61 +7,14 @@ #include "EnumUtility.h" #include "LoggingModule.h" #include +#include #include -#include namespace Aws { namespace IoTFleetWise { -void -CollectionSchemeManager::prepareCheckinTimer() -{ - auto currTime = mClock->timeSinceEpoch(); - TimeData checkinData = TimeData{ currTime, CHECKIN }; - mTimeLine.push( checkinData ); -} - -bool -CollectionSchemeManager::sendCheckin() -{ - // Create a list of active collectionSchemes and the current decoder manifest and send it to cloud - std::vector checkinMsg; - for ( auto it = mEnabledCollectionSchemeMap.begin(); it != mEnabledCollectionSchemeMap.end(); it++ ) - { - checkinMsg.emplace_back( it->first ); - } - for ( auto it = mIdleCollectionSchemeMap.begin(); it != mIdleCollectionSchemeMap.end(); it++ ) - { - checkinMsg.emplace_back( it->first ); - } - if ( !mCurrentDecoderManifestID.empty() ) - { - checkinMsg.emplace_back( mCurrentDecoderManifestID ); - } - std::string checkinLogStr; - for ( size_t i = 0; i < checkinMsg.size(); i++ ) - { - if ( i > 0 ) - { - checkinLogStr += ", "; - } - checkinLogStr += checkinMsg[i]; - } - FWE_LOG_TRACE( "CHECKIN: " + checkinLogStr ); - - if ( mSchemaListenerPtr == nullptr ) - { - FWE_LOG_ERROR( "Cannot set the checkin message" ); - return false; - } - else - { - return mSchemaListenerPtr->sendCheckin( checkinMsg ); - } -} - bool CollectionSchemeManager::retrieve( DataType retrieveType ) { @@ -73,7 +26,7 @@ CollectionSchemeManager::retrieve( DataType retrieveType ) if ( mSchemaPersistency == nullptr ) { - FWE_LOG_ERROR( "Failed to acquire a valid handle on the scheme local persistency module" ); + FWE_LOG_INFO( "Persistency module not available" ); return false; } switch ( retrieveType ) @@ -87,7 +40,7 @@ CollectionSchemeManager::retrieve( DataType retrieveType ) errStr = "Failed to retrieve the DecoderManifest from the persistency module due to an error: "; break; default: - FWE_LOG_ERROR( "Unknown error: " + std::to_string( toUType( retrieveType ) ) ); + FWE_LOG_ERROR( "Unknown data type: " + std::to_string( toUType( retrieveType ) ) ); return false; } @@ -117,11 +70,8 @@ CollectionSchemeManager::retrieve( DataType retrieveType ) mCollectionSchemeList->copyData( protoOutput.data(), protoSize ); mProcessCollectionScheme = true; } - // currently this if will be always true as it can be only DECODER_MANIFEST or COLLECTION_SCHEME_LIST but for - // readability leave it as else if instead of else - // coverity[autosar_cpp14_m0_1_2_violation] - // coverity[autosar_cpp14_m0_1_9_violation] - // coverity[misra_cpp_2008_rule_0_1_9_violation] + // coverity[autosar_cpp14_m0_1_9_violation] - Second if-statement always follows same path as first + // coverity[misra_cpp_2008_rule_0_1_9_violation] - Second if-statement always follows same path as first else if ( retrieveType == DataType::DECODER_MANIFEST ) { // updating mDecoderManifest @@ -144,7 +94,7 @@ CollectionSchemeManager::store( DataType storeType ) if ( mSchemaPersistency == nullptr ) { - FWE_LOG_ERROR( "Failed to acquire a valid handle on the scheme local persistency module" ); + FWE_LOG_INFO( "Persistency module not available" ); return; } if ( ( storeType == DataType::COLLECTION_SCHEME_LIST ) && ( mCollectionSchemeList == nullptr ) ) diff --git a/src/QueueTypes.h b/src/QueueTypes.h new file mode 100644 index 00000000..7b1865c2 --- /dev/null +++ b/src/QueueTypes.h @@ -0,0 +1,153 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "Listener.h" +#include "LoggingModule.h" +#include "TraceModule.h" +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +// Thread-safe queue with mutex +template +struct LockedQueue +{ +public: + /** + * @param maxSize the maximum number of elements that can be stored in the queue. If reached, any new data will be + * discarded. + * @param queueName a name to identify the queue, mostly for logging and metrics purposes. + * @param traceMetric the metric that should be emitted when the number of elements change. + * @param notifyOnEveryNumOfElements notify the listeners only when the queue grows by this number of elements. + * This is useful to avoid very busy queues notify the listeners too often. This way, the queue can notify + * listeners only when data is being produced much faster than consumed. + */ + LockedQueue( size_t maxSize, + std::string queueName, + const boost::optional traceMetric = boost::none, + size_t notifyOnEveryNumOfElements = 1 ) + : mMaxSize( maxSize ) + , mQueueName( std::move( queueName ) ) + , mTraceMetric( traceMetric ) + , mNotifyOnEveryNumOfElements( notifyOnEveryNumOfElements ) + { + } + + bool + push( const T &element ) + { + std::lock_guard lock( mMutex ); + if ( ( mQueue.size() + 1 ) > mMaxSize ) + { + FWE_LOG_WARN( "Queue " + mQueueName + " is full" ) + return false; + } + mQueue.push( element ); + if ( mTraceMetric.has_value() ) + { + TraceModule::get().incrementAtomicVariable( mTraceMetric.get() ); + } + + if ( ( mQueue.size() % mNotifyOnEveryNumOfElements ) == 0 ) + { + mListeners.notify(); + } + + return true; + } + + bool + pop( T &element ) + { + std::lock_guard lock( mMutex ); + if ( mQueue.empty() ) + { + return false; + } + element = mQueue.front(); + mQueue.pop(); + if ( mTraceMetric.has_value() ) + { + TraceModule::get().decrementAtomicVariable( mTraceMetric.get() ); + } + return true; + } + + template + size_t + consumeAll( const Functor &functor ) + { + size_t consumed = 0; + T element; + while ( pop( element ) ) + { + functor( element ); + consumed++; + } + return consumed; + } + + /** + * @brief Register a callback to be called when new data is pushed to the queue + * + * This can be used, for example, to wake up a thread that is interested in processing this data. + */ + void + subscribeToNewDataAvailable( std::function callback ) + { + mListeners.subscribe( callback ); + } + + // coverity[misra_cpp_2008_rule_14_7_1_violation] Required in unit tests + bool + isEmpty() + { + std::lock_guard lock( mMutex ); + return mQueue.empty(); + } + +private: + std::mutex mMutex; + size_t mMaxSize; + std::queue mQueue; + ThreadSafeListeners> mListeners; + std::string mQueueName; + boost::optional mTraceMetric; + size_t mNotifyOnEveryNumOfElements; +}; + +// This queue wrapper is used to support one/multiple producers to multiple queues logic +// It will push an element to all registered thread-safe queues +template +struct LockedQueueDistributor +{ +public: + void + push( const T &&element ) + { + for ( auto queue : mLockedQueues ) + { + queue->push( element ); + } + } + void + registerQueue( const std::shared_ptr> queuePtr ) + { + mLockedQueues.push_back( queuePtr ); + } + +private: + std::vector>> mLockedQueues; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/ROS2DataSource.cpp b/src/ROS2DataSource.cpp index d2b56ae7..47163e84 100644 --- a/src/ROS2DataSource.cpp +++ b/src/ROS2DataSource.cpp @@ -3,7 +3,7 @@ #include "ROS2DataSource.h" #include "LoggingModule.h" -#include "TraceModule.h" +#include "QueueTypes.h" #include #include #include @@ -117,10 +117,10 @@ ROS2DataSourceNode::subscribe( std::string topic, } ROS2DataSource::ROS2DataSource( ROS2DataSourceConfig config, - SignalBufferPtr signalBufferPtr, + SignalBufferDistributorPtr signalBufferDistributor, std::shared_ptr rawDataBufferManager ) : mConfig( std::move( config ) ) - , mSignalBufferPtr( std::move( signalBufferPtr ) ) + , mSignalBufferDistributor( std::move( signalBufferDistributor ) ) , mRawBufferManager( std::move( rawDataBufferManager ) ) { mNode = std::make_shared(); @@ -150,7 +150,7 @@ ROS2DataSource::onChangeOfActiveDictionary( ConstDecoderDictionaryConstPtr &dict // Name the executer threads the first time they process a callback. Until then they will have the name of the main // thread fwVNROS2main static thread_local bool gAlreadyNamedThread{ false }; // NOLINT Thread local thread named flag -static std::atomic gThreadCounter{ 0 }; // NOLINT Global atomic thread counter +static std::atomic gThreadCounter{ 1 }; // NOLINT Global atomic thread counter /* * called from multiple threads in parallel as callback group is Reentrant @@ -243,24 +243,14 @@ ROS2DataSource::topicCallback( std::shared_ptr msg, s messageFormat.mSignalId, bufferHandle, RawData::BufferHandleUsageStage::COLLECTED_NOT_IN_HISTORY_BUFFER ); - auto collectedSignal = CollectedSignal( - messageFormat.mSignalId, timestamp, bufferHandle, SignalType::RAW_DATA_BUFFER_HANDLE ); + auto collectedSignal = + CollectedSignal( messageFormat.mSignalId, timestamp, bufferHandle, SignalType::COMPLEX_SIGNAL ); collectedSignalsGroup.push_back( collectedSignal ); } } } - TraceModule::get().incrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - TraceModule::get().addToAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignalsGroup.size() ); - if ( !mSignalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ) - { - TraceModule::get().decrementAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES ); - - TraceModule::get().subtractFromAtomicVariable( TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_SIGNALS, - collectedSignalsGroup.size() ); - FWE_LOG_WARN( "Signal Buffer Full" ); - } + mSignalBufferDistributor->push( CollectedDataFrame( collectedSignalsGroup ) ); } void @@ -279,9 +269,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, bool b = false; // coverity[misra_cpp_2008_rule_14_8_2_violation] - FastCDR header defines both template and type overloads cdr >> b; - collectedSignal = CollectedSignal( - signalId, timestamp, b, SignalType::DOUBLE ); // TODO: replace DOUBLE with format.mPrimitiveType as soon as - // collection inspection engine fully supports different types + collectedSignal = CollectedSignal( signalId, timestamp, b, format.mPrimitiveType ); } break; case SignalType::UINT8: { @@ -290,7 +278,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> u8; u8 = static_cast( static_cast( format.mOffset ) + u8 * static_cast( format.mScaling ) ); - collectedSignal = CollectedSignal( signalId, timestamp, u8, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, u8, format.mPrimitiveType ); } break; case SignalType::UINT16: { @@ -299,7 +287,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> u16; u16 = static_cast( static_cast( format.mOffset ) + u16 * static_cast( format.mScaling ) ); - collectedSignal = CollectedSignal( signalId, timestamp, u16, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, u16, format.mPrimitiveType ); } break; case SignalType::UINT32: { @@ -308,7 +296,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> u32; u32 *= static_cast( format.mScaling ); u32 += static_cast( format.mOffset ); - collectedSignal = CollectedSignal( signalId, timestamp, u32, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, u32, format.mPrimitiveType ); } break; case SignalType::UINT64: { @@ -317,7 +305,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> u64; u64 *= static_cast( format.mScaling ); u64 += static_cast( format.mOffset ); - collectedSignal = CollectedSignal( signalId, timestamp, u64, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, u64, format.mPrimitiveType ); } break; case SignalType::INT8: { @@ -325,7 +313,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, // coverity[misra_cpp_2008_rule_14_8_2_violation] - FastCDR header defines both template and type overloads cdr >> i8; i8 = static_cast( static_cast( format.mOffset ) + i8 * static_cast( format.mScaling ) ); - collectedSignal = CollectedSignal( signalId, timestamp, i8, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, i8, format.mPrimitiveType ); } break; case SignalType::INT16: { @@ -334,7 +322,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> i16; i16 = static_cast( static_cast( format.mOffset ) + i16 * static_cast( format.mScaling ) ); - collectedSignal = CollectedSignal( signalId, timestamp, i16, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, i16, format.mPrimitiveType ); } break; case SignalType::INT32: { @@ -343,7 +331,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> i32; i32 *= static_cast( format.mScaling ); i32 += static_cast( format.mOffset ); - collectedSignal = CollectedSignal( signalId, timestamp, i32, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, i32, format.mPrimitiveType ); } break; case SignalType::INT64: { @@ -352,7 +340,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> i64; i64 *= static_cast( format.mScaling ); i64 += static_cast( format.mOffset ); - collectedSignal = CollectedSignal( signalId, timestamp, i64, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, i64, format.mPrimitiveType ); } break; case SignalType::FLOAT: { @@ -361,7 +349,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> f; f *= static_cast( format.mScaling ); f += static_cast( format.mOffset ); - collectedSignal = CollectedSignal( signalId, timestamp, f, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, f, format.mPrimitiveType ); } break; case SignalType::DOUBLE: { @@ -370,7 +358,7 @@ ROS2DataSource::readPrimitiveDataFromCDR( eprosima::fastcdr::Cdr &cdr, cdr >> d; d *= format.mScaling; d += format.mOffset; - collectedSignal = CollectedSignal( signalId, timestamp, d, SignalType::DOUBLE ); + collectedSignal = CollectedSignal( signalId, timestamp, d, format.mPrimitiveType ); } break; default: @@ -1107,6 +1095,7 @@ ROS2DataSource::doWork( void *data ) { // coverity[autosar_cpp14_a15_5_2_violation] false positive - join is called to exit the previous thread // coverity[cert_err50_cpp_violation] false positive - join is called to exit the previous thread + gThreadCounter = 1; ros2DataSource->mExecutorSpinThread = std::thread( [ros2DataSource]() { FWE_LOG_TRACE( "Executor start spinning" ); ros2DataSource->mROS2Executor->spin(); diff --git a/src/ROS2DataSource.h b/src/ROS2DataSource.h index e5b9f36b..acc49155 100644 --- a/src/ROS2DataSource.h +++ b/src/ROS2DataSource.h @@ -100,7 +100,7 @@ class ROS2DataSource { public: ROS2DataSource( ROS2DataSourceConfig config, - SignalBufferPtr signalBufferPtr, + SignalBufferDistributorPtr signalBufferDistributor, std::shared_ptr rawDataBufferManager = nullptr ); ~ROS2DataSource(); @@ -328,7 +328,7 @@ class ROS2DataSource mutable std::mutex mThreadMutex; Signal mWait; std::shared_ptr mClock = ClockHandler::getClock(); - SignalBufferPtr mSignalBufferPtr; + SignalBufferDistributorPtr mSignalBufferDistributor; std::mutex mDecoderDictMutex; std::atomic mEventNewDecoderManifestAvailable{ false }; diff --git a/src/RawDataManager.cpp b/src/RawDataManager.cpp index ea9b01fa..91f58a4a 100644 --- a/src/RawDataManager.cpp +++ b/src/RawDataManager.cpp @@ -146,7 +146,7 @@ BufferManager::updateConfig( const std::unordered_map lock( mBufferManagerMutex ); @@ -664,9 +664,11 @@ BufferManager::Buffer::deleteUnusedData() TraceModule::get().incrementVariable( TraceVariable::RAW_DATA_OVERWRITTEN_DATA_WITH_USED_HANDLE ); } } + // coverity[unchecked_value] The return value gives the status and does not need to be checked FWE_LOG_TRACE( "Deleting data for Signal ID " + std::to_string( mTypeID ) + " BufferHandle " + std::to_string( unusedDataFrame->mHandleID ) + - " with usage hints: " + std::to_string( unusedDataFrame->hasUsageHints() ) ); + " with usage hints: " + std::to_string( unusedDataFrame->hasUsageHints() ) + + ", total bytes in use before deletion: " + std::to_string( mBytesInUse ) ); auto deletedMemSize = unusedDataFrame->mRawData.size(); mBytesInUse -= deletedMemSize; // Subtract the memory mBuffer.erase( unusedDataFrame ); // Delete the data @@ -679,7 +681,8 @@ size_t BufferManager::Buffer::deleteDataFromHandle( const BufferHandle handle ) { FWE_LOG_TRACE( "Deleting data for Signal ID " + std::to_string( mTypeID ) + " BufferHandle " + - std::to_string( handle ) ); + std::to_string( handle ) + + ", total bytes in use before deletion: " + std::to_string( mBytesInUse ) ); auto unusedDataFrame = std::find_if( mBuffer.cbegin(), mBuffer.cend(), [&]( const Frame &rawDataFrame ) -> bool { return rawDataFrame.mHandleID == handle; } ); diff --git a/src/RawDataManager.h b/src/RawDataManager.h index 9c4f8c1b..77a25a93 100644 --- a/src/RawDataManager.h +++ b/src/RawDataManager.h @@ -251,6 +251,8 @@ struct SignalBufferOverrides }; // coverity[cert_dcl60_cpp_violation] false positive - class only defined once +// coverity[autosar_cpp14_m3_2_2_violation] false positive - class only defined once +// coverity[misra_cpp_2008_rule_3_2_2_violation ] false positive - class only defined once class BufferManagerConfig { public: @@ -317,7 +319,7 @@ class BufferManagerConfig size_t mMaxBytesPerSample{ 0 }; size_t mMaxBytesPerSignal{ 0 }; - std::unordered_map> + std::unordered_map> mOverridesPerSignal; // It can contain config overrides for a specific signal. When a signal is not // present in this map, the default config (defined by the other members) is used. The // first key is the interfaceId and the second key is the messageId. Both are needed to @@ -400,7 +402,7 @@ class BufferManager * to this handle will be kept. It may need to be released to give space to more recent data. * When the data needs to be read, the handle should be passed to the borrowFrame method. */ - virtual BufferHandle push( uint8_t *data, size_t size, Timestamp receiveTimestamp, BufferTypeId typeId ); + virtual BufferHandle push( const uint8_t *data, size_t size, Timestamp receiveTimestamp, BufferTypeId typeId ); /** * @brief Get the Statistics for a particular signal raw buffer @@ -430,7 +432,7 @@ class BufferManager * @param handle Unique handle of the raw data requested * @return LoanedFrame */ - LoanedFrame borrowFrame( BufferTypeId typeId, BufferHandle handle ); + virtual LoanedFrame borrowFrame( BufferTypeId typeId, BufferHandle handle ); /** * @brief Mark a handle as being used and for what purpose. diff --git a/src/RemoteProfiler.cpp b/src/RemoteProfiler.cpp index a0831843..b7c48503 100644 --- a/src/RemoteProfiler.cpp +++ b/src/RemoteProfiler.cpp @@ -54,12 +54,13 @@ RemoteProfiler::sendMetricsOut() Json::StreamWriterBuilder builder; builder["indentation"] = ""; // If you want whitespace-less output const std::string output = Json::writeString( builder, fMetricsRoot ); - uint32_t ret = static_cast( - fMetricsSender->sendBuffer( reinterpret_cast( output.c_str() ), output.length() ) ); - if ( static_cast( ConnectivityError::Success ) != ret ) - { - FWE_LOG_ERROR( "Send error " + std::to_string( static_cast( ret ) ) ); - } + fMetricsSender->sendBuffer( + reinterpret_cast( output.c_str() ), output.length(), []( ConnectivityError result ) { + if ( result == ConnectivityError::Success ) + { + FWE_LOG_ERROR( "Send error " + std::to_string( static_cast( result ) ) ); + } + } ); fMetricsRoot.clear(); fCurrentMetricsPending = 0; } @@ -81,12 +82,13 @@ RemoteProfiler::sendLogsOut() if ( ( fLogSender != nullptr ) ) { - uint32_t ret = static_cast( - fLogSender->sendBuffer( reinterpret_cast( output.c_str() ), output.length() ) ); - if ( static_cast( ConnectivityError::Success ) != ret ) - { - FWE_LOG_ERROR( " Send error " + std::to_string( static_cast( ret ) ) ); - } + fLogSender->sendBuffer( + reinterpret_cast( output.c_str() ), output.length(), []( ConnectivityError result ) { + if ( result == ConnectivityError::Success ) + { + FWE_LOG_ERROR( "Send error " + std::to_string( static_cast( result ) ) ); + } + } ); } } } @@ -103,14 +105,15 @@ RemoteProfiler::logMessage( LogLevel level, return; } Json::Value logNode; - + const std::string timestamp = fClock->currentTimeToIsoString(); logNode["logLevel"] = levelToString( level ); logNode["logFile"] = filename; logNode["logLineNumber"] = lineNumber; logNode["logFunction"] = function; logNode["logEntry"] = logEntry; + logNode["logTimestamp"] = timestamp; uint32_t size = static_cast( filename.length() + function.length() + logEntry.length() + - JSON_MAX_OVERHEAD_BYTES_PER_LOG ); + timestamp.length() + JSON_MAX_OVERHEAD_BYTES_PER_LOG ); bool sendOutBeforeAdding = false; { std::lock_guard lock( loggingMutex ); diff --git a/src/RemoteProfiler.h b/src/RemoteProfiler.h index a6f56a2f..a1c4f9f3 100644 --- a/src/RemoteProfiler.h +++ b/src/RemoteProfiler.h @@ -40,8 +40,8 @@ class RemoteProfiler : public IMetricsReceiver, public ILogger /** * @brief Construct the RemoteProfiler which can upload metrics and logs over MQTT * - * @param metricsSender the channel that should be used to upload metrics - * @param logSender the channel that should be used to upload logs + * @param metricsSender the sender that should be used to upload metrics + * @param logSender the sender that should be used to upload logs * @param initialMetricsUploadInterval the interval used between two metrics uploads * @param initialLogMaxInterval the max interval that logs can be cached before uploading * @param initialLogLevelThresholdToSend all logs below this threshold will be ignored diff --git a/src/RetryThread.cpp b/src/RetryThread.cpp index 69a48963..f23f60f7 100644 --- a/src/RetryThread.cpp +++ b/src/RetryThread.cpp @@ -5,6 +5,7 @@ #include "LoggingModule.h" #include #include +#include namespace Aws { @@ -13,15 +14,16 @@ namespace IoTFleetWise std::atomic RetryThread::fInstanceCounter( 0 ); // NOLINT Global atomic instance counter -RetryThread::RetryThread( IRetryable &retryable, uint32_t startBackoffMs, uint32_t maxBackoffMs ) - : fRetryable( retryable ) - // coverity[misra_cpp_2008_rule_5_2_10_violation] For std::atomic this must be performed in a single statement - // coverity[autosar_cpp14_m5_2_10_violation] For std::atomic this must be performed in a single statement +// coverity[misra_cpp_2008_rule_5_2_10_violation] For std::atomic this must be performed in a single statement +// coverity[autosar_cpp14_m5_2_10_violation] For std::atomic this must be performed in a single statement +RetryThread::RetryThread( Retryable retryable, uint32_t startBackoffMs, uint32_t maxBackoffMs ) + : fRetryable( std::move( retryable ) ) , fInstance( fInstanceCounter++ ) , fStartBackoffMs( startBackoffMs ) , fMaxBackoffMs( maxBackoffMs ) , fCurrentWaitTime( 0 ) , fShouldStop( false ) + , fRestart( false ) { } @@ -46,6 +48,13 @@ RetryThread::start() return fThread.isValid(); } +void +RetryThread::restart() +{ + fRestart.store( true ); + fWait.notify(); +} + bool RetryThread::stop() { @@ -68,23 +77,37 @@ RetryThread::doWork( void *data ) { RetryThread *retryThread = static_cast( data ); retryThread->fCurrentWaitTime = retryThread->fStartBackoffMs; + RetryStatus result = RetryStatus::RETRY; while ( !retryThread->fShouldStop ) { - RetryStatus result = retryThread->fRetryable.attempt(); - if ( result != RetryStatus::RETRY ) + if ( ( result == RetryStatus::RETRY ) || retryThread->fRestart ) + { + if ( retryThread->fRestart ) + { + retryThread->fCurrentWaitTime = retryThread->fStartBackoffMs; + retryThread->fRestart.store( false, std::memory_order_relaxed ); + } + + result = retryThread->fRetryable(); + if ( result != RetryStatus::RETRY ) + { + FWE_LOG_TRACE( "Finished with code " + std::to_string( static_cast( result ) ) ); + retryThread->fWait.wait( Signal::WaitWithPredicate ); + continue; + } + + FWE_LOG_TRACE( "Current retry time is: " + std::to_string( retryThread->fCurrentWaitTime ) ); + retryThread->fWait.wait( retryThread->fCurrentWaitTime ); + // exponential backoff + retryThread->fCurrentWaitTime = std::min( retryThread->fCurrentWaitTime * 2, retryThread->fMaxBackoffMs ); + } + else { - FWE_LOG_TRACE( "Finished with code " + std::to_string( static_cast( result ) ) ); - retryThread->fRetryable.onFinished( result ); - return; + retryThread->fWait.wait( Signal::WaitWithPredicate ); } - FWE_LOG_TRACE( "Current retry time is: " + std::to_string( retryThread->fCurrentWaitTime ) ); - retryThread->fWait.wait( retryThread->fCurrentWaitTime ); - // exponential backoff - retryThread->fCurrentWaitTime = std::min( retryThread->fCurrentWaitTime * 2, retryThread->fMaxBackoffMs ); } // If thread is shutdown without succeeding signal abort FWE_LOG_TRACE( "Stop thread with ABORT" ); - retryThread->fRetryable.onFinished( RetryStatus::ABORT ); } } // namespace IoTFleetWise diff --git a/src/RetryThread.h b/src/RetryThread.h index 77e098c3..30701708 100644 --- a/src/RetryThread.h +++ b/src/RetryThread.h @@ -7,6 +7,7 @@ #include "Thread.h" #include #include +#include #include namespace Aws @@ -21,27 +22,16 @@ enum class RetryStatus ABORT }; -class IRetryable -{ -public: - /** - * @brief This function will be called for every retry - * @return decides if the function can be retried later or succeeded or unrecoverable failed - */ - virtual RetryStatus attempt() = 0; - - /** - * @brief Is called after the retries stopped which means it succeeded or is aborted - * @param code signals how it finished: if it was aborted or succeeded. retry should be never observed here - */ - virtual void onFinished( RetryStatus code ) = 0; - virtual ~IRetryable() = default; -}; +/** + * @brief This function will be called for every retry + * @return decides if the function can be retried later or succeeded or unrecoverable failed + */ +using Retryable = std::function; class RetryThread { public: - RetryThread( IRetryable &retryable, uint32_t startBackoffMs, uint32_t maxBackoffMs ); + RetryThread( Retryable retryable, uint32_t startBackoffMs, uint32_t maxBackoffMs ); /** * @brief start the thread @@ -49,6 +39,15 @@ class RetryThread */ bool start(); + /** + * @brief Restart the thread + * + * If the retryable is currently running, it won't be stopped, but the thread will run it again + * even if the retryable succeeds. + * The time to wait between retries is also reset. + */ + void restart(); + /** * @brief stops the thread * @return true if thread stopping was successful @@ -83,13 +82,14 @@ class RetryThread static void doWork( void *data ); static std::atomic fInstanceCounter; // NOLINT Global atomic instance counter - IRetryable &fRetryable; + Retryable fRetryable; int fInstance; const uint32_t fStartBackoffMs; const uint32_t fMaxBackoffMs; uint32_t fCurrentWaitTime; Thread fThread; std::atomic fShouldStop; + std::atomic fRestart; std::mutex fThreadMutex; Signal fWait; }; diff --git a/src/S3Sender.cpp b/src/S3Sender.cpp index 9f7f804e..439e5c00 100644 --- a/src/S3Sender.cpp +++ b/src/S3Sender.cpp @@ -3,13 +3,15 @@ #include "S3Sender.h" #include "LoggingModule.h" +#include "TraceModule.h" +#include #include +#include #include #include #include #include #include -#include #include #include #include @@ -36,15 +38,8 @@ transferStatusToString( Aws::Transfer::TransferStatus transferStatus ) return ss.str(); } -S3Sender::S3Sender( - - std::shared_ptr payloadManager, - std::function( - Aws::Client::ClientConfiguration &clientConfiguration, - Aws::Transfer::TransferManagerConfiguration &transferManagerConfiguration )> createTransferManagerWrapper, - size_t multipartSize ) +S3Sender::S3Sender( CreateTransferManagerWrapper createTransferManagerWrapper, size_t multipartSize ) : mMultipartSize{ multipartSize == 0 ? DEFAULT_MULTIPART_SIZE : multipartSize } - , mPayloadManager( std::move( payloadManager ) ) , mCreateTransferManagerWrapper( std::move( createTransferManagerWrapper ) ) { } @@ -81,36 +76,43 @@ S3Sender::disconnect() return true; } -ConnectivityError +void S3Sender::sendStream( std::unique_ptr streambufBuilder, const S3UploadMetadata &uploadMetadata, const std::string &objectKey, - std::function resultCallback ) + ResultCallback resultCallback ) { if ( streambufBuilder == nullptr ) { FWE_LOG_ERROR( "No valid streambuf builder provided for the upload" ); - return ConnectivityError::WrongInputData; + resultCallback( ConnectivityError::WrongInputData, nullptr ); + return; } if ( mCreateTransferManagerWrapper == nullptr ) { FWE_LOG_ERROR( "No S3 client configured" ); - persistS3Request( std::move( streambufBuilder ), objectKey ); - return ConnectivityError::NotConfigured; + // TODO: Once we move the simultaneous uploads limit to DataSenderManager, we won't need to + // pass the streambuf via callback, because VisionSystemDataSender will already pass the + // streambuf instead of the StreambufBuilder to S3Sender. So VisionSystemDataSender will already + // have a reference/pointer to the data. + // coverity[autosar_cpp14_a20_8_6_violation] no way to construct it with make_shared + std::shared_ptr streambuf = streambufBuilder->build(); + resultCallback( ConnectivityError::NotConfigured, streambuf ); + return; } { std::lock_guard lock( mQueuedAndOngoingUploadsLookupMutex ); - FWE_LOG_INFO( "Queuing async upload for object " + objectKey + " to the bucket " + uploadMetadata.bucketName ); + FWE_LOG_INFO( "Queuing async upload for object " + objectKey + " to the bucket " + uploadMetadata.bucketName + + " , Current queue size: " + std::to_string( mQueuedUploads.size() ) ); mQueuedUploads.push( { std::move( streambufBuilder ), uploadMetadata, objectKey, resultCallback } ); + TraceModule::get().setVariable( TraceVariable::QUEUED_S3_OBJECTS, mQueuedUploads.size() ); } submitQueuedUploads(); - - return ConnectivityError::Success; -} // namespace IoTFleetWise +} void S3Sender::submitQueuedUploads() @@ -119,17 +121,20 @@ S3Sender::submitQueuedUploads() while ( ( mOngoingUploads.size() < MAX_SIMULTANEOUS_FILES ) && ( !mQueuedUploads.empty() ) ) { + ResultCallback resultCallback; QueuedUploadMetadata &queuedUploadMetadata = mQueuedUploads.front(); auto streambuf = queuedUploadMetadata.streambufBuilder->build(); auto uploadMetadata = queuedUploadMetadata.uploadMetadata; auto objectKey = queuedUploadMetadata.objectKey; - auto resultCallback = queuedUploadMetadata.resultCallback; + resultCallback = queuedUploadMetadata.resultCallback; mQueuedUploads.pop(); + TraceModule::get().setVariable( TraceVariable::QUEUED_S3_OBJECTS, mQueuedUploads.size() ); if ( streambuf == nullptr ) { FWE_LOG_WARN( "Skipping upload of object " + objectKey + " to the bucket " + uploadMetadata.bucketName + " because its data is not available anymore" ); + resultCallback( ConnectivityError::WrongInputData, nullptr ); continue; } @@ -141,6 +146,7 @@ S3Sender::submitQueuedUploads() { FWE_LOG_ERROR( "Could not prepare data for the upload of object " + objectKey + " to the bucket " + uploadMetadata.bucketName ); + resultCallback( ConnectivityError::WrongInputData, nullptr ); continue; } @@ -200,6 +206,7 @@ S3Sender::getTransferManagerWrapper( const S3UploadMetadata &uploadMetadata ) void S3Sender::transferStatusUpdatedCallback( const std::shared_ptr &transferHandle ) { + // coverity[check_return] Status can be used directly without being checked auto transferStatus = transferHandle->GetStatus(); FWE_LOG_TRACE( "Transfer status for object " + transferHandle->GetKey() + " updated: " + transferStatusToString( transferStatus ) + @@ -256,46 +263,18 @@ S3Sender::transferStatusUpdatedCallback( const std::shared_ptr resultCallback; + ResultCallback resultCallback; + std::shared_ptr streambuf; { std::lock_guard lock( mQueuedAndOngoingUploadsLookupMutex ); resultCallback = mOngoingUploads[transferHandle->GetKey()].resultCallback; + streambuf = mOngoingUploads[transferHandle->GetKey()].streambuf; mOngoingUploads.erase( transferHandle->GetKey() ); } submitQueuedUploads(); - if ( resultCallback ) - { - resultCallback( result ); - } -} - -void -S3Sender::persistS3Request( std::unique_ptr streambufBuilder, std::string objectKey ) -{ - // TODO: persist/retry on fail. Retry will only be executed as upload of the persisted data. - // Temp change: dump data from stream into the file here - if ( mPayloadManager == nullptr ) - { - FWE_LOG_WARN( "Could not persist data for object " + objectKey + " : Payload manager is not configured." ); - return; - } - if ( streambufBuilder == nullptr ) - { - FWE_LOG_WARN( "Could not persist data for object " + objectKey + " : Invalid streambuf builder provided." ); - return; - } - - auto streambuf = streambufBuilder->build(); - if ( streambuf == nullptr ) - { - FWE_LOG_WARN( "Could not persist data for object " + objectKey + " : Invalid stream." ); - return; - } - - streambuf->pubseekoff( std::streambuf::off_type( 0 ), std::ios_base::beg, std::ios_base::in ); - mPayloadManager->storeIonData( std::move( streambuf ), objectKey ); + resultCallback( result ? ConnectivityError::Success : ConnectivityError::TransmissionError, streambuf ); } } // namespace IoTFleetWise diff --git a/src/S3Sender.h b/src/S3Sender.h index 17e7c31c..617c9e30 100644 --- a/src/S3Sender.h +++ b/src/S3Sender.h @@ -5,12 +5,8 @@ #include "ICollectionScheme.h" #include "IConnectionTypes.h" -#include "PayloadManager.h" #include "StreambufBuilder.h" #include "TransferManagerWrapper.h" -#include -#include -#include #include #include #include @@ -33,18 +29,13 @@ namespace IoTFleetWise class S3Sender { public: + using ResultCallback = std::function data )>; + /** - * @param payloadManager the payload manager to be used when data needs to be persisted. nullptr - * can be passed. In such case, data won't be persisted after a failure. * @param createTransferManagerWrapper a factory function that creates a new Transfer Manager instance * @param multipartSize the size that will be used to decide whether to try a multipart upload */ - S3Sender( - std::shared_ptr payloadManager, - std::function( - Aws::Client::ClientConfiguration &clientConfiguration, - Aws::Transfer::TransferManagerConfiguration &transferManagerConfiguration )> createTransferManagerWrapper, - size_t multipartSize ); + S3Sender( CreateTransferManagerWrapper createTransferManagerWrapper, size_t multipartSize ); virtual ~S3Sender() = default; S3Sender() = delete; @@ -55,41 +46,37 @@ class S3Sender virtual bool disconnect(); - virtual ConnectivityError sendStream( std::unique_ptr streambufBuilder, - const S3UploadMetadata &uploadMetadata, - const std::string &objectKey, - std::function resultCallback ); + virtual void sendStream( std::unique_ptr streambufBuilder, + const S3UploadMetadata &uploadMetadata, + const std::string &objectKey, + ResultCallback resultCallback ); private: size_t mMultipartSize; std::unordered_map> mRegionToTransferManagerWrapper; - std::shared_ptr mPayloadManager; - std::function( - Aws::Client::ClientConfiguration &clientConfiguration, - Aws::Transfer::TransferManagerConfiguration &transferManagerConfiguration )> - mCreateTransferManagerWrapper; + CreateTransferManagerWrapper mCreateTransferManagerWrapper; struct OngoingUploadMetadata { std::shared_ptr streambuf; - std::function resultCallback; + ResultCallback resultCallback; std::shared_ptr transferManagerWrapper; std::shared_ptr transferHandle; uint8_t attempts; }; std::mutex mQueuedAndOngoingUploadsLookupMutex; - std::unordered_map mOngoingUploads; + std::unordered_map mOngoingUploads; struct QueuedUploadMetadata { std::unique_ptr streambufBuilder; S3UploadMetadata uploadMetadata; std::string objectKey; - std::function resultCallback; + ResultCallback resultCallback; }; std::queue mQueuedUploads; /** - * @brief TODO + * @brief check the ongoing uploads and submit uploads from the queue if possible */ void submitQueuedUploads(); @@ -109,14 +96,6 @@ class S3Sender * the status of a transfer changes. */ void transferStatusUpdatedCallback( const std::shared_ptr &transferHandle ); - - /** - * @brief Pass data stream to the payload manager to persist - * - * @param streambufBuilder object that can create the stream with data to persist - * @param objectKey S3 object key - */ - void persistS3Request( std::unique_ptr streambufBuilder, std::string objectKey ); }; } // namespace IoTFleetWise diff --git a/src/Schema.cpp b/src/Schema.cpp index c3aabcef..410db378 100644 --- a/src/Schema.cpp +++ b/src/Schema.cpp @@ -17,12 +17,12 @@ Schema::Schema( std::shared_ptr receiverDecoderManifest, : mSender( std::move( sender ) ) { // Register the listeners - receiverDecoderManifest->subscribeToDataReceived( [this]( const ReceivedChannelMessage &receivedChannelMessage ) { - onDecoderManifestReceived( receivedChannelMessage.buf, receivedChannelMessage.size ); + receiverDecoderManifest->subscribeToDataReceived( [this]( const ReceivedConnectivityMessage &receivedMessage ) { + onDecoderManifestReceived( receivedMessage.buf, receivedMessage.size ); } ); receiverCollectionSchemeList->subscribeToDataReceived( - [this]( const ReceivedChannelMessage &receivedChannelMessage ) { - onCollectionSchemeReceived( receivedChannelMessage.buf, receivedChannelMessage.size ); + [this]( const ReceivedConnectivityMessage &receivedMessage ) { + onCollectionSchemeReceived( receivedMessage.buf, receivedMessage.size ); } ); } @@ -32,7 +32,7 @@ Schema::onDecoderManifestReceived( const uint8_t *buf, size_t size ) // Check for a empty input data if ( ( buf == nullptr ) || ( size == 0 ) ) { - FWE_LOG_ERROR( "Received empty CollectionScheme List data from Cloud" ); + FWE_LOG_ERROR( "Received empty Decoder Manifest List data from the Cloud" ); return; } @@ -75,8 +75,8 @@ Schema::onCollectionSchemeReceived( const uint8_t *buf, size_t size ) FWE_LOG_TRACE( "Received CollectionSchemeList" ); } -bool -Schema::sendCheckin( const std::vector &documentARNs ) +void +Schema::sendCheckin( const std::vector &documentARNs, OnCheckinSentCallback callback ) { mProtoCheckinMsg.Clear(); @@ -92,59 +92,60 @@ Schema::sendCheckin( const std::vector &documentARNs ) if ( !mProtoCheckinMsg.SerializeToString( &mProtoCheckinMsgOutput ) ) { FWE_LOG_ERROR( "Checkin serialization failed" ); - return false; - } - else - { - // transmit the data to the cloud - FWE_LOG_TRACE( "Sending a Checkin message to the backend" ); - return transmitCheckin(); + callback( false ); + return; } + + // transmit the data to the cloud + FWE_LOG_TRACE( "Sending a Checkin message to the backend" ); + transmitCheckin( callback ); } -bool -Schema::transmitCheckin() +void +Schema::transmitCheckin( OnCheckinSentCallback callback ) { if ( mSender == nullptr ) { FWE_LOG_ERROR( "Invalid sender instance" ); - return false; + callback( false ); + return; } - auto res = mSender->sendBuffer( reinterpret_cast( mProtoCheckinMsgOutput.data() ), - mProtoCheckinMsgOutput.size() ); + // Trace log for more verbose Checkin Info + std::string checkinDebugString; + checkinDebugString = "Checkin data: timestamp: " + std::to_string( mProtoCheckinMsg.timestamp_ms_epoch() ); + checkinDebugString += " with " + std::to_string( mProtoCheckinMsg.document_sync_ids_size() ) + " documents: ["; - if ( res == ConnectivityError::Success ) + for ( int i = 0; i < mProtoCheckinMsg.document_sync_ids_size(); i++ ) { - FWE_LOG_TRACE( "Checkin Message sent to the backend" ); - - // Trace log for more verbose Checkin Info - std::string checkinDebugString; - checkinDebugString = "Checkin data: timestamp: " + std::to_string( mProtoCheckinMsg.timestamp_ms_epoch() ); - checkinDebugString += " with " + std::to_string( mProtoCheckinMsg.document_sync_ids_size() ) + " documents: ["; - - for ( int i = 0; i < mProtoCheckinMsg.document_sync_ids_size(); i++ ) + if ( i > 0 ) { - if ( i > 0 ) - { - checkinDebugString += ", "; - } - checkinDebugString += mProtoCheckinMsg.document_sync_ids( i ); + checkinDebugString += ", "; } - checkinDebugString += "]"; - - FWE_LOG_TRACE( checkinDebugString ); - return true; - } - else if ( res == ConnectivityError::NoConnection ) - { - return false; - } - else - { - FWE_LOG_ERROR( "offboardconnectivity error, will retry sending the checkin message" ); - return false; + checkinDebugString += mProtoCheckinMsg.document_sync_ids( i ); } + checkinDebugString += "]"; + + mSender->sendBuffer( reinterpret_cast( mProtoCheckinMsgOutput.data() ), + mProtoCheckinMsgOutput.size(), + [checkinDebugString, callback]( ConnectivityError result ) { + if ( result == ConnectivityError::Success ) + { + FWE_LOG_TRACE( "Checkin Message sent to the backend" ); + FWE_LOG_TRACE( checkinDebugString ); + callback( true ); + } + else if ( result == ConnectivityError::NoConnection ) + { + FWE_LOG_TRACE( "Couldn't send checkin message because there is no connection" ); + callback( false ); + } + else + { + FWE_LOG_ERROR( "offboardconnectivity error, will retry sending the checkin message" ); + callback( false ); + } + } ); } } // namespace IoTFleetWise diff --git a/src/Schema.h b/src/Schema.h index fe309ebf..d41fae6e 100644 --- a/src/Schema.h +++ b/src/Schema.h @@ -13,6 +13,7 @@ #include "ISender.h" #include "Listener.h" #include "SchemaListener.h" +#include "SignalTypes.h" #include "checkin.pb.h" #include #include @@ -89,9 +90,11 @@ class Schema : public SchemaListener * system. * * @param documentARNs List of the ARNs - * @return True if the message has been sent. False otherwise. + * @param callback callback that will be called when the operation completes (successfully or not). + * IMPORTANT: The callback can be called by the same thread before sendBuffer even returns + * or a separate thread, depending on whether the results are known synchronously or asynchronously. */ - bool sendCheckin( const std::vector &documentARNs ) override; + void sendCheckin( const std::vector &documentARNs, OnCheckinSentCallback callback ) override; private: /** @@ -133,10 +136,11 @@ class Schema : public SchemaListener /** * @brief Sends an mProtoCheckinMsgOutput string on the checkin topic - * @return True if the Connectivity Module packed and send the data out of the process space. - * It does not guarantee that the data reached the Checkin topic ( best effort QoS ) + * @param callback callback that will be called when the operation completes (successfully or not). + * IMPORTANT: The callback can be called by the same thread before sendBuffer even returns + * or a separate thread, depending on whether the results are known synchronously or asynchronously. */ - bool transmitCheckin(); + void transmitCheckin( OnCheckinSentCallback callback ); }; } // namespace IoTFleetWise diff --git a/src/SchemaListener.h b/src/SchemaListener.h index 6ae504b8..7a820416 100644 --- a/src/SchemaListener.h +++ b/src/SchemaListener.h @@ -3,6 +3,8 @@ #pragma once +#include "SignalTypes.h" +#include #include #include @@ -19,15 +21,19 @@ class SchemaListener public: virtual ~SchemaListener() = default; + using OnCheckinSentCallback = std::function; + /** * @brief Callback implementation for receiving sendCheckin requests from CollectionScheme Management * * @param documentARNs A list containing loaded collectionScheme ID's in the form of ARNs (both active and inactive) * and the loaded Decoder Manifest ID in the form of ARN if one is present. Documents do not need to be in any * order. - * @return True if the message has been sent. False otherwise. + * @param callback callback that will be called when the operation completes (successfully or not). + * IMPORTANT: The callback can be called by the same thread before sendBuffer even returns + * or a separate thread, depending on whether the results are known synchronously or asynchronously. */ - virtual bool sendCheckin( const std::vector &documentARNs ) = 0; + virtual void sendCheckin( const std::vector &documentARNs, OnCheckinSentCallback callback ) = 0; }; } // namespace IoTFleetWise diff --git a/src/SignalTypes.h b/src/SignalTypes.h index b4ec7d8f..e054e355 100644 --- a/src/SignalTypes.h +++ b/src/SignalTypes.h @@ -35,12 +35,18 @@ static constexpr CANChannelNumericID INVALID_CAN_SOURCE_NUMERIC_ID = 0xFFFFFFFF; using InterfaceID = std::string; static const InterfaceID INVALID_INTERFACE_ID{}; +/** + * @brief The sync ID is a concatenation of the resource ARN, a slash '/', and a timestamp. + * This ID is used to uniquely identify and synchronize documents between the cloud and edge. + */ +using SyncID = std::string; + /** * @brief Signal ID is either an ID provided by Cloud that is unique across all signals found in the vehicle regardless - * of network bus or an internal ID see INTERNAL_SIGNAL_ID_BITMASK + * of network bus or an internal ID see INTERNAL_SIGNAL_ID_BITMASK. Cloud starts allocating signal IDs starting at 1. */ using SignalID = uint32_t; -static constexpr SignalID INVALID_SIGNAL_ID = 0xFFFFFFFF; +static constexpr SignalID INVALID_SIGNAL_ID = 0; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA /** @@ -79,11 +85,57 @@ enum struct SignalType FLOAT = 8, DOUBLE = 9, BOOLEAN = 10, + UNKNOWN = 11, #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - RAW_DATA_BUFFER_HANDLE = 11, // internal type RawData::BufferHandle is defined as uint32 + COMPLEX_SIGNAL = 12, // internal type RawData::BufferHandle is defined as uint32 #endif }; +/** + * @brief Converts SignalType to a string to be used in logs + * + * @param signalType The signal type to be converted. + * @return A string describing the signal type + */ +inline std::string +signalTypeToString( SignalType signalType ) +{ + // coverity[autosar_cpp14_m6_4_6_violation] + // coverity[misra_cpp_2008_rule_6_4_6_violation] compiler warning is preferred over a default-clause + switch ( signalType ) + { + case SignalType::UINT8: + return "UINT8"; + case SignalType::INT8: + return "INT8"; + case SignalType::UINT16: + return "UINT16"; + case SignalType::INT16: + return "INT16"; + case SignalType::UINT32: + return "UINT32"; + case SignalType::INT32: + return "INT32"; + case SignalType::UINT64: + return "UINT64"; + case SignalType::INT64: + return "INT64"; + case SignalType::FLOAT: + return "FLOAT"; + case SignalType::DOUBLE: + return "DOUBLE"; + case SignalType::BOOLEAN: + return "BOOLEAN"; + case SignalType::UNKNOWN: + return "UNKNOWN"; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + return "COMPLEX_SIGNAL"; +#endif + } + return "UNREACHABLE"; +} + /** * @brief Format that defines a CAN Signal Format */ @@ -130,9 +182,9 @@ struct CANSignalFormat bool mIsMultiplexorSignal{ false }; /** - * @brief The datatype of the signal. The default is double for backward compatibility + * @brief The datatype of the signal. */ - SignalType mSignalType{ SignalType::DOUBLE }; + SignalType mSignalType{ SignalType::UNKNOWN }; /** * @brief If mIsMultiplexorSignal is true, this value will be the value e.g. m0. If false, the value will be maxbit8 @@ -176,7 +228,43 @@ struct CANSignalFormat } }; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +union DecodedSignalValueType { + double doubleVal; + uint64_t uint64Val; + int64_t int64Val; +}; + +/** + * @brief A signal value that was decoded by a data source + * + * The actual value can be of different types, but we don't consider all possible supported types + * here. To simplify the decoding only the largest types are used to avoid losing precision. + * Since this value is not intended to be stored for a long time, there is not much impact e.g. using + * a double for a uint8_t. + */ +struct DecodedSignalValue +{ + DecodedSignalValueType signalValue; + SignalType signalType; + + template + DecodedSignalValue( T val, SignalType type ) + : signalType( type ) + { + switch ( signalType ) + { + case SignalType::UINT64: + signalValue.uint64Val = static_cast( val ); + break; + case SignalType::INT64: + signalValue.int64Val = static_cast( val ); + break; + default: + signalValue.doubleVal = static_cast( val ); + } + } +}; + namespace RawData { @@ -184,7 +272,6 @@ using BufferHandle = uint32_t; static constexpr BufferHandle INVALID_BUFFER_HANDLE = 0; } // namespace RawData -#endif } // namespace IoTFleetWise } // namespace Aws diff --git a/src/TelemetryDataSender.cpp b/src/TelemetryDataSender.cpp new file mode 100644 index 00000000..a58bbcdc --- /dev/null +++ b/src/TelemetryDataSender.cpp @@ -0,0 +1,433 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "TelemetryDataSender.h" +#include "IConnectionTypes.h" +#include "LoggingModule.h" +#include "OBDDataTypes.h" +#include "SignalTypes.h" +#include "TraceModule.h" +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +constexpr uint32_t MAX_NUMBER_OF_SIGNAL_TO_TRACE_LOG = 6; + +class TelemetryDataToPersist : public DataToPersist +{ +public: + TelemetryDataToPersist( const CollectionSchemeParams &collectionSchemeParams, + unsigned partNumber, + std::shared_ptr data ) + : mCollectionSchemeParams( collectionSchemeParams ) + , mPartNumber( partNumber ) + , mData( std::move( data ) ) + { + } + + SenderDataType + getDataType() const override + { + return SenderDataType::TELEMETRY; + } + + Json::Value + getMetadata() const override + { + Json::Value metadata; + metadata["compressionRequired"] = mCollectionSchemeParams.compression; + return metadata; + } + + std::string + getFilename() const override + { + return std::to_string( mCollectionSchemeParams.eventID ) + "-" + + std::to_string( mCollectionSchemeParams.triggerTime ) + "-" + std::to_string( mPartNumber ) + ".bin"; + }; + + boost::variant, std::shared_ptr> + getData() const override + { + return mData; + } + +private: + CollectionSchemeParams mCollectionSchemeParams; + unsigned mPartNumber; + std::shared_ptr mData; +}; + +TelemetryDataSender::TelemetryDataSender( std::shared_ptr mqttSender, + std::shared_ptr protoWriter, + PayloadAdaptionConfig configUncompressed, + PayloadAdaptionConfig configCompressed ) + : mMQTTSender( std::move( mqttSender ) ) + , mProtoWriter( std::move( protoWriter ) ) + , mConfigUncompressed( configUncompressed ) + , mConfigCompressed( configCompressed ) +{ + mConfigUncompressed.transmitSizeThreshold = + ( mMQTTSender->getMaxSendSize() * mConfigUncompressed.transmitThresholdStartPercent ) / 100; + mConfigCompressed.transmitSizeThreshold = + ( mMQTTSender->getMaxSendSize() * mConfigCompressed.transmitThresholdStartPercent ) / 100; +} + +void +TelemetryDataSender::processData( std::shared_ptr data, OnDataProcessedCallback callback ) +{ + if ( data == nullptr ) + { + FWE_LOG_WARN( "Nothing to send as the input is empty" ); + return; + } + + auto triggeredCollectionSchemeDataPtr = std::dynamic_pointer_cast( data ); + if ( triggeredCollectionSchemeDataPtr == nullptr ) + { + FWE_LOG_WARN( "Nothing to send as the input is not a valid TriggeredCollectionSchemeData" ); + return; + } + + std::string firstSignalValues = "["; + uint32_t signalPrintCounter = 0; + std::string firstSignalTimestamp; + for ( auto &s : triggeredCollectionSchemeDataPtr->signals ) + { + if ( firstSignalTimestamp.empty() ) + { + firstSignalTimestamp = " first signal timestamp: " + std::to_string( s.receiveTime ); + } + signalPrintCounter++; + if ( signalPrintCounter > MAX_NUMBER_OF_SIGNAL_TO_TRACE_LOG ) + { + firstSignalValues += " ..."; + break; + } + auto signalValue = s.getValue(); + firstSignalValues += std::to_string( s.signalID ) + ":"; + switch ( signalValue.getType() ) + { + case SignalType::UINT8: + firstSignalValues += std::to_string( signalValue.value.uint8Val ) + ","; + break; + case SignalType::INT8: + firstSignalValues += std::to_string( signalValue.value.int8Val ) + ","; + break; + case SignalType::UINT16: + firstSignalValues += std::to_string( signalValue.value.uint16Val ) + ","; + break; + case SignalType::INT16: + firstSignalValues += std::to_string( signalValue.value.int16Val ) + ","; + break; + case SignalType::UINT32: + firstSignalValues += std::to_string( signalValue.value.uint32Val ) + ","; + break; + case SignalType::INT32: + firstSignalValues += std::to_string( signalValue.value.int32Val ) + ","; + break; + case SignalType::UINT64: + firstSignalValues += std::to_string( signalValue.value.uint64Val ) + ","; + break; + case SignalType::INT64: + firstSignalValues += std::to_string( signalValue.value.int64Val ) + ","; + break; + case SignalType::FLOAT: + firstSignalValues += std::to_string( signalValue.value.floatVal ) + ","; + break; + case SignalType::DOUBLE: + firstSignalValues += std::to_string( signalValue.value.doubleVal ) + ","; + break; + case SignalType::BOOLEAN: + firstSignalValues += std::to_string( static_cast( signalValue.value.boolVal ) ) + ","; + break; + case SignalType::UNKNOWN: + // Signal of type UNKNOWN cannot be processed + break; +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + case SignalType::COMPLEX_SIGNAL: + firstSignalValues += std::to_string( signalValue.value.uint32Val ) + ","; + break; +#endif + } + } + firstSignalValues += "]"; + // Avoid invoking Data Collection Sender if there is nothing to send. + if ( triggeredCollectionSchemeDataPtr->signals.empty() && triggeredCollectionSchemeDataPtr->canFrames.empty() && + triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes.empty() +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + && triggeredCollectionSchemeDataPtr->uploadedS3Objects.empty() +#endif + ) + { + FWE_LOG_INFO( "The trigger for Campaign: " + triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID + + " activated eventID: " + std::to_string( triggeredCollectionSchemeDataPtr->eventID ) + + " but no data is available to ingest" ); + } + else + { + std::string message = + "FWE data ready to send with eventID " + std::to_string( triggeredCollectionSchemeDataPtr->eventID ) + + " from " + triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID + + " Signals:" + std::to_string( triggeredCollectionSchemeDataPtr->signals.size() ) + " " + firstSignalValues + + firstSignalTimestamp + + " trigger timestamp: " + std::to_string( triggeredCollectionSchemeDataPtr->triggerTime ) + + " raw CAN frames:" + std::to_string( triggeredCollectionSchemeDataPtr->canFrames.size() ) + + " DTCs:" + std::to_string( triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes.size() ) +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + + " Uploaded S3 Objects: " + std::to_string( triggeredCollectionSchemeDataPtr->uploadedS3Objects.size() ) +#endif + ; + FWE_LOG_INFO( message ); + + setCollectionSchemeParameters( triggeredCollectionSchemeDataPtr ); + transformTelemetryDataToProto( triggeredCollectionSchemeDataPtr, callback ); + } +} + +void +TelemetryDataSender::setCollectionSchemeParameters( + std::shared_ptr &triggeredCollectionSchemeDataPtr ) +{ + mCollectionSchemeParams.persist = triggeredCollectionSchemeDataPtr->metadata.persist; + mCollectionSchemeParams.compression = triggeredCollectionSchemeDataPtr->metadata.compress; + mCollectionSchemeParams.priority = triggeredCollectionSchemeDataPtr->metadata.priority; + mCollectionSchemeParams.eventID = triggeredCollectionSchemeDataPtr->eventID; + mCollectionSchemeParams.triggerTime = triggeredCollectionSchemeDataPtr->triggerTime; +} + +void +TelemetryDataSender::transformTelemetryDataToProto( + std::shared_ptr &triggeredCollectionSchemeDataPtr, + OnDataProcessedCallback callback ) +{ + // Clear old data and setup metadata + mPartNumber = 0; + mProtoWriter->setupVehicleData( triggeredCollectionSchemeDataPtr, mCollectionSchemeParams.eventID ); + + // Iterate through all the signals and add to the protobuf + for ( const auto &signal : triggeredCollectionSchemeDataPtr->signals ) + { +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // Filter out the raw data and internal signals + if ( ( signal.value.type != SignalType::COMPLEX_SIGNAL ) && + ( ( signal.signalID & INTERNAL_SIGNAL_ID_BITMASK ) == 0 ) ) +#endif + { + appendMessageToProto( triggeredCollectionSchemeDataPtr, signal, callback ); + } + } + + // Iterate through all the raw CAN frames and add to the protobuf + for ( const auto &canFrame : triggeredCollectionSchemeDataPtr->canFrames ) + { + appendMessageToProto( triggeredCollectionSchemeDataPtr, canFrame, callback ); + } + + // Add DTC info to the payload + if ( triggeredCollectionSchemeDataPtr->mDTCInfo.hasItems() ) + { + mProtoWriter->setupDTCInfo( triggeredCollectionSchemeDataPtr->mDTCInfo ); + const auto &dtcCodes = triggeredCollectionSchemeDataPtr->mDTCInfo.mDTCCodes; + + // Iterate through all the DTC codes and add to the protobuf + for ( const auto &dtc : dtcCodes ) + { + appendMessageToProto( triggeredCollectionSchemeDataPtr, dtc, callback ); + } + } + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + for ( const auto &object : triggeredCollectionSchemeDataPtr->uploadedS3Objects ) + { + appendMessageToProto( triggeredCollectionSchemeDataPtr, object, callback ); + } +#endif + + // Serialize and transmit any remaining messages + if ( mProtoWriter->isVehicleDataAdded() ) + { + uploadProto( callback ); + } +} + +bool +TelemetryDataSender::serialize( std::string &output ) +{ + if ( !mProtoWriter->serializeVehicleData( &output ) ) + { + FWE_LOG_ERROR( "Serialization failed" ); + return false; + } + return true; +} + +bool +TelemetryDataSender::compress( std::string &input, std::string &output ) const +{ + if ( mCollectionSchemeParams.compression ) + { + FWE_LOG_TRACE( "Compress the payload before transmitting since compression flag is true" ); + if ( snappy::Compress( input.data(), input.size(), &output ) == 0U ) + { + FWE_LOG_TRACE( "Error in compressing the payload" ); + return false; + } + } + return true; +} + +size_t +TelemetryDataSender::uploadProto( OnDataProcessedCallback callback, unsigned recursionLevel ) +{ + if ( mMQTTSender == nullptr ) + { + FWE_LOG_ERROR( "No sender provided" ); + return 0; + } + + auto protoOutput = std::make_shared(); + + if ( !serialize( *protoOutput ) ) + { + FWE_LOG_ERROR( "Data cannot be uploaded due to serialization failure" ); + return 0; + } + + if ( mCollectionSchemeParams.compression ) + { + auto compressedProtoOutput = std::make_shared(); + if ( !compress( *protoOutput, *compressedProtoOutput ) ) + { + FWE_LOG_ERROR( "Data cannot be uploaded due to compression failure" ); + return 0; + } + protoOutput = compressedProtoOutput; + } + + auto maxSendSize = mMQTTSender->getMaxSendSize(); + auto &config = mCollectionSchemeParams.compression ? mConfigCompressed : mConfigUncompressed; + auto payloadSizeLimitMax = ( maxSendSize * config.payloadSizeLimitMaxPercent ) / 100; + if ( protoOutput->size() > payloadSizeLimitMax ) + { + config.transmitSizeThreshold = + ( config.transmitSizeThreshold * ( 100 - config.transmitThresholdAdaptPercent ) ) / 100; + FWE_LOG_TRACE( "Payload size " + std::to_string( protoOutput->size() ) + " above maximum limit " + + std::to_string( payloadSizeLimitMax ) + ". Decreasing " + + ( mCollectionSchemeParams.compression ? "compressed" : "uncompressed" ) + + " transmit threshold by " + std::to_string( config.transmitThresholdAdaptPercent ) + "% to " + + std::to_string( config.transmitSizeThreshold ) ); + } + if ( protoOutput->size() > maxSendSize ) + { + if ( recursionLevel >= UPLOAD_PROTO_RECURSION_LIMIT ) + { + // If this happens frequently, look if a new repeated field type was added to the VehicleData protobuf + // schema, but was not added to splitVehicleData and mergeVehicleData. This would cause the code below to + // not actually make the payload any smaller. + FWE_LOG_ERROR( "Payload dropped as it could not be split smaller than maximum payload size." ); + return 0; + } + FWE_LOG_TRACE( + "Payload size " + std::to_string( protoOutput->size() ) + " exceeds the maximum payload size of " + + std::to_string( maxSendSize ) + " for " + + ( mCollectionSchemeParams.compression ? "compressed" : "uncompressed" ) + + " data. Attempting to split in half and try again. Recursion level: " + std::to_string( recursionLevel ) ); + Schemas::VehicleDataMsg::VehicleData data; + mProtoWriter->splitVehicleData( data ); + uploadProto( callback, recursionLevel + 1 ); + mProtoWriter->mergeVehicleData( data ); + uploadProto( callback, recursionLevel + 1 ); + return protoOutput->size(); + } + + mPartNumber++; + mMQTTSender->sendBuffer( + reinterpret_cast( protoOutput->data() ), + protoOutput->size(), + [callback, + collectionSchemeParams = mCollectionSchemeParams, + partNumber = mPartNumber, + protoOutput, + vehicleDataEstimatedSize = mProtoWriter->getVehicleDataEstimatedSize(), + threshold = config.transmitSizeThreshold]( ConnectivityError result ) { + if ( result == ConnectivityError::Success ) + { + FWE_LOG_INFO( "Payload has been uploaded, size: " + std::to_string( protoOutput->size() ) + + " bytes, part number: " + std::to_string( partNumber ) + + ", compressed: " + std::to_string( collectionSchemeParams.compression ) + + ", vehicle data estimated size: " + std::to_string( vehicleDataEstimatedSize ) + + ", transmit size threshold: " + std::to_string( threshold ) ); + TraceModule::get().addToVariable( TraceVariable::DATA_FORWARD_BYTES, protoOutput->size() ); + TraceModule::get().incrementVariable( TraceVariable::VEHICLE_DATA_PUBLISH_COUNT ); + callback( true, nullptr ); + } + else + { + std::shared_ptr dataToPersist; + if ( collectionSchemeParams.persist ) + { + dataToPersist = + std::make_shared( collectionSchemeParams, partNumber, protoOutput ); + } + callback( false, dataToPersist ); + } + } ); + + TraceModule::get().sectionEnd( TraceSection::COLLECTION_SCHEME_CHANGE_TO_FIRST_DATA ); + TraceModule::get().incrementVariable( TraceVariable::MQTT_SIGNAL_MESSAGES_SENT_OUT ); + + return protoOutput->size(); +} + +void +TelemetryDataSender::processPersistedData( std::istream &data, + const Json::Value &metadata, + OnPersistedDataProcessedCallback callback ) +{ + static_cast( metadata ); + + if ( !mMQTTSender->isAlive() ) + { + callback( false ); + return; + } + + data.seekg( 0, std::ios::end ); + auto size = data.tellg(); + auto dataAsArray = std::vector( static_cast( size ) ); + data.seekg( 0, std::ios::beg ); + data.read( dataAsArray.data(), static_cast( size ) ); + + if ( !data.good() ) + { + FWE_LOG_ERROR( "Failed to read persisted data" ); + callback( false ); + return; + } + + auto buf = reinterpret_cast( dataAsArray.data() ); + auto bufSize = static_cast( size ); + mMQTTSender->sendBuffer( buf, bufSize, [callback, size]( ConnectivityError result ) { + if ( result != ConnectivityError::Success ) + { + callback( false ); + return; + } + + FWE_LOG_INFO( "A Payload of size: " + std::to_string( size ) + " bytes has been uploaded" ); + callback( true ); + } ); +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/TelemetryDataSender.h b/src/TelemetryDataSender.h new file mode 100644 index 00000000..f89c4e29 --- /dev/null +++ b/src/TelemetryDataSender.h @@ -0,0 +1,139 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "CollectionInspectionAPITypes.h" +#include "DataSenderProtoWriter.h" +#include "DataSenderTypes.h" +#include "ISender.h" +#include "LoggingModule.h" +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +struct PayloadAdaptionConfig +{ + // Starting transmit threshold as percent of maximum payload size: + unsigned transmitThresholdStartPercent{}; + // Adapt transmit threshold if payload size is outside this percentage range of the maximum payload size: + unsigned payloadSizeLimitMinPercent{}; + unsigned payloadSizeLimitMaxPercent{}; + // Adapt transmit threshold by this amount when it is outside the range: + unsigned transmitThresholdAdaptPercent{}; + // Transmit size treshold is set by TelemetryDataSender: + size_t transmitSizeThreshold{}; +}; + +class TelemetryDataSender : public DataSender +{ + +public: + TelemetryDataSender( std::shared_ptr mqttSender, + std::shared_ptr protoWriter, + PayloadAdaptionConfig configUncompressed, + PayloadAdaptionConfig configCompressed ); + + ~TelemetryDataSender() override = default; + + void processData( std::shared_ptr data, OnDataProcessedCallback callback ) override; + + void processPersistedData( std::istream &data, + const Json::Value &metadata, + OnPersistedDataProcessedCallback callback ) override; + +private: + std::shared_ptr mMQTTSender; + std::shared_ptr mProtoWriter; + PayloadAdaptionConfig mConfigUncompressed; + PayloadAdaptionConfig mConfigCompressed; + + CollectionSchemeParams mCollectionSchemeParams; + unsigned mPartNumber{ 0 }; // track how many payloads the data was split in + + /** + * @brief Set up collectionSchemeParams struct + * @param triggeredCollectionSchemeDataPtr collected data + */ + void setCollectionSchemeParameters( + std::shared_ptr &triggeredCollectionSchemeDataPtr ); + + /** + * @brief Put collected telemetry data into protobuf in chunks. Initiates serialization, compression, and + * upload for each partition. + * @param triggeredCollectionSchemeDataPtr collected data + * @param callback callback function to be called after data is processed + */ + void transformTelemetryDataToProto( + std::shared_ptr &triggeredCollectionSchemeDataPtr, + OnDataProcessedCallback callback ); + + /** + * @brief If the serialized payload ends up larger than the maximum payload size, it will be split in half in a + * recursive manner and re-serialized. This constant limits the number of times it will be split. I.e. 2 means it + * first tries splitting in half, then if that's still too large it will try splitting into quarters before dropping + * the payload. + */ + static constexpr unsigned UPLOAD_PROTO_RECURSION_LIMIT = 2; + + /** + * @brief Serializes, compresses, and uploads proto output. + * @param callback Callback called after upload success / failure + * @param recursionLevel The level of recursion + * @return payload size in bytes or zero on error + */ + size_t uploadProto( OnDataProcessedCallback callback, unsigned recursionLevel = 0 ); + + template + void + appendMessageToProto( std::shared_ptr &triggeredCollectionSchemeDataPtr, + T msg, + OnDataProcessedCallback callback ) + { + mProtoWriter->append( msg ); + auto &config = triggeredCollectionSchemeDataPtr->metadata.compress ? mConfigCompressed : mConfigUncompressed; + if ( mProtoWriter->getVehicleDataEstimatedSize() >= config.transmitSizeThreshold ) + { + auto payloadSize = uploadProto( callback ); + auto maxSendSize = mMQTTSender->getMaxSendSize(); + auto payloadSizeLimitMin = ( maxSendSize * config.payloadSizeLimitMinPercent ) / 100; + if ( ( payloadSize > 0 ) && ( payloadSize < payloadSizeLimitMin ) ) + { + config.transmitSizeThreshold = + ( config.transmitSizeThreshold * ( 100 + config.transmitThresholdAdaptPercent ) ) / 100; + FWE_LOG_TRACE( "Payload size " + std::to_string( payloadSize ) + " below minimum limit " + + std::to_string( payloadSizeLimitMin ) + ". Increasing " + + ( triggeredCollectionSchemeDataPtr->metadata.compress ? "compressed" : "uncompressed" ) + + " transmit threshold by " + std::to_string( config.transmitThresholdAdaptPercent ) + + "% to " + std::to_string( config.transmitSizeThreshold ) ); + } + // Setup the next payload chunk + mProtoWriter->setupVehicleData( triggeredCollectionSchemeDataPtr, mCollectionSchemeParams.eventID ); + } + } + + /** + * @brief Serializes data + * @param output Output string + * @return True if serialization succeeds + */ + bool serialize( std::string &output ); + + /** + * @brief Compresses data + * @param input Input data string + * @param output Where the result of compression will be saved to + * @return True if compression succeeds + */ + bool compress( std::string &input, std::string &output ) const; +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/TraceModule.cpp b/src/TraceModule.cpp index 7500fb43..3a6b5f9e 100644 --- a/src/TraceModule.cpp +++ b/src/TraceModule.cpp @@ -97,8 +97,6 @@ TraceModule::getVariableName( TraceVariable variable ) return "RFrames18_id18"; case TraceVariable::READ_SOCKET_FRAMES_19: return "RFrames19_id19"; - case TraceVariable::QUEUE_INSPECTION_TO_SENDER: - return "QStS_id40"; case TraceVariable::MAX_SYSTEMTIME_KERNELTIME_DIFF: return "SysKerTimeDiff_id41"; case TraceVariable::PM_MEMORY_NULL: @@ -153,10 +151,16 @@ TraceModule::getVariableName( TraceVariable variable ) return "RawElementsPerType"; case TraceVariable::RAW_DATA_BUFFER_MANAGER_BYTES: return "RawBufferBytes"; + case TraceVariable::QUEUED_S3_OBJECTS: + return "QueuedS3Objects"; case TraceVariable::CE_PROCESSED_DATA_FRAMES: return "CEProcessedDataFrames"; case TraceVariable::CE_PROCESSED_DTCS: return "CEProcessedDTCs"; + case TraceVariable::DATA_FORWARD_BYTES: + return "DataForwardBytes"; + case TraceVariable::VEHICLE_DATA_PUBLISH_COUNT: + return "VehicleDataPublishCount"; // Intentionally omit default so that we can use compiler warnings to remind us about missing values } return nullptr; @@ -179,6 +183,8 @@ TraceModule::getAtomicVariableName( TraceAtomicVariable variable ) return "QueueConsumerToInspectionDataFrames"; case TraceAtomicVariable::QUEUE_CONSUMER_TO_INSPECTION_DTCS: return "QueueConsumerToInspectionDtcs"; + case TraceAtomicVariable::QUEUE_INSPECTION_TO_SENDER: + return "QStS_id40"; case TraceAtomicVariable::NOT_TIME_MONOTONIC_FRAMES: return "nTime_id2"; case TraceAtomicVariable::SUBSCRIBE_ERROR: diff --git a/src/TraceModule.h b/src/TraceModule.h index 01a8a5c5..44ecd828 100644 --- a/src/TraceModule.h +++ b/src/TraceModule.h @@ -42,7 +42,6 @@ enum class TraceVariable READ_SOCKET_FRAMES_17, READ_SOCKET_FRAMES_18, READ_SOCKET_FRAMES_19, // If you add more, update references to this - QUEUE_INSPECTION_TO_SENDER, MAX_SYSTEMTIME_KERNELTIME_DIFF, PM_MEMORY_NULL, PM_MEMORY_INSUFFICIENT, @@ -72,6 +71,9 @@ enum class TraceVariable RAW_DATA_OVERWRITTEN_DATA_WITH_USED_HANDLE, RAW_DATA_BUFFER_ELEMENTS_PER_TYPE, RAW_DATA_BUFFER_MANAGER_BYTES, + QUEUED_S3_OBJECTS, + DATA_FORWARD_BYTES, + VEHICLE_DATA_PUBLISH_COUNT, // If you add more, remember to add the name to TraceModule::getVariableName TRACE_VARIABLE_SIZE }; @@ -85,6 +87,7 @@ enum class TraceAtomicVariable QUEUE_CONSUMER_TO_INSPECTION_CAN, QUEUE_CONSUMER_TO_INSPECTION_DATA_FRAMES, QUEUE_CONSUMER_TO_INSPECTION_DTCS, + QUEUE_INSPECTION_TO_SENDER, NOT_TIME_MONOTONIC_FRAMES, SUBSCRIBE_ERROR, SUBSCRIBE_REJECT, diff --git a/src/TransferManagerWrapper.h b/src/TransferManagerWrapper.h index 9342b481..2f2b67b9 100644 --- a/src/TransferManagerWrapper.h +++ b/src/TransferManagerWrapper.h @@ -3,7 +3,10 @@ #pragma once +#include #include +#include +#include namespace Aws { @@ -83,5 +86,9 @@ class TransferManagerWrapper std::shared_ptr mTransferManager; }; +using CreateTransferManagerWrapper = std::function( + Aws::Client::ClientConfiguration &clientConfiguration, + Aws::Transfer::TransferManagerConfiguration &transferManagerConfiguration )>; + } // namespace IoTFleetWise } // namespace Aws diff --git a/src/VisionSystemDataSender.cpp b/src/VisionSystemDataSender.cpp new file mode 100644 index 00000000..7d098a24 --- /dev/null +++ b/src/VisionSystemDataSender.cpp @@ -0,0 +1,256 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "VisionSystemDataSender.h" +#include "CollectionInspectionAPITypes.h" +#include "IConnectionTypes.h" +#include "LoggingModule.h" +#include "QueueTypes.h" +#include "SignalTypes.h" +#include "StreambufBuilder.h" +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +class VisionSystemDataToPersist : public DataToPersist +{ +public: + VisionSystemDataToPersist( std::string objectKey, + // coverity[pass_by_value] conflicts with clang-tidy's modernize-pass-by-value rule + S3UploadParams s3UploadParams, + std::shared_ptr data ) + : mObjectKey( std::move( objectKey ) ) + , mS3UploadParams( std::move( s3UploadParams ) ) + , mData( std::move( data ) ) + { + } + + SenderDataType + getDataType() const override + { + return SenderDataType::VISION_SYSTEM; + } + + Json::Value + getMetadata() const override + { + // VisionSystemData persistency is not fully supported yet. When it is this should contain + // additional metadata to allow retrying this data. + Json::Value metadata; + return metadata; + } + + std::string + getFilename() const override + { + return mObjectKey; + } + + boost::variant, std::shared_ptr> + getData() const override + { + return mData; + } + +private: + std::string mObjectKey; + S3UploadParams mS3UploadParams; + std::shared_ptr mData; +}; + +VisionSystemDataSender::VisionSystemDataSender( std::shared_ptr uploadedS3Objects, + std::shared_ptr s3Sender, + std::shared_ptr ionWriter, + std::string vehicleName ) + : mUploadedS3Objects( std::move( uploadedS3Objects ) ) + , mIonWriter( std::move( ionWriter ) ) + , mS3Sender{ std::move( s3Sender ) } + , mVehicleName( std::move( vehicleName ) ) + +{ +} + +void +VisionSystemDataSender::processData( std::shared_ptr data, OnDataProcessedCallback callback ) +{ + if ( data == nullptr ) + { + FWE_LOG_WARN( "Nothing to send as the input is empty" ); + return; + } + + auto triggeredVisionSystemData = std::dynamic_pointer_cast( data ); + if ( triggeredVisionSystemData == nullptr ) + { + FWE_LOG_WARN( "Nothing to send as the input is not a valid TriggeredVisionSystemData" ); + return; + } + + std::string firstSignalValues = "["; + uint32_t signalPrintCounter = 0; + uint32_t maxNumberOfSignalsToTrace = 6; + std::string firstSignalTimestamp; + for ( auto &s : triggeredVisionSystemData->signals ) + { + if ( firstSignalTimestamp.empty() ) + { + firstSignalTimestamp = " first signal timestamp: " + std::to_string( s.receiveTime ); + } + signalPrintCounter++; + if ( signalPrintCounter > maxNumberOfSignalsToTrace ) + { + firstSignalValues += " ..."; + break; + } + auto signalValue = s.getValue(); + firstSignalValues += std::to_string( s.signalID ) + ":"; + switch ( signalValue.getType() ) + { + case SignalType::COMPLEX_SIGNAL: + firstSignalValues += std::to_string( signalValue.value.uint32Val ) + ","; + break; + default: + // Only raw data is used for VisionSystemData + break; + } + } + firstSignalValues += "]"; + + FWE_LOG_INFO( + "FWE VisionSystemData ready to be sent with eventID " + std::to_string( triggeredVisionSystemData->eventID ) + + " from " + triggeredVisionSystemData->metadata.collectionSchemeID + + " Signals:" + std::to_string( triggeredVisionSystemData->signals.size() ) + " " + firstSignalValues + + firstSignalTimestamp + " trigger timestamp: " + std::to_string( triggeredVisionSystemData->triggerTime ) ); + + transformVisionSystemDataToIon( triggeredVisionSystemData, callback ); +} + +void +VisionSystemDataSender::onChangeCollectionSchemeList( + const std::shared_ptr &activeCollectionSchemes ) +{ + FWE_LOG_INFO( "New active collection scheme list was handed over to Data Sender" ); + std::lock_guard lock( mActiveCollectionSchemeMutex ); + mActiveCollectionSchemes = activeCollectionSchemes; +} + +S3UploadMetadata +VisionSystemDataSender::getS3UploadMetadataForCollectionScheme( const std::string &collectionSchemeID ) +{ + std::lock_guard lock( mActiveCollectionSchemeMutex ); + if ( mActiveCollectionSchemes != nullptr ) + { + for ( const auto &scheme : mActiveCollectionSchemes->activeCollectionSchemes ) + { + if ( scheme->getCollectionSchemeID() == collectionSchemeID ) + { + return scheme->getS3UploadMetadata(); + } + } + } + return {}; +} + +void +VisionSystemDataSender::transformVisionSystemDataToIon( + std::shared_ptr triggeredVisionSystemData, OnDataProcessedCallback callback ) +{ + if ( triggeredVisionSystemData->signals.empty() ) + { + return; + } + if ( mS3Sender == nullptr ) + { + FWE_LOG_ERROR( "Can not send data to S3 as S3Sender is not initalized. Please make sure config parameters in " + "section credentialsProvider are correct" ); + return; + } + if ( mIonWriter == nullptr ) + { + FWE_LOG_WARN( "IonWriter is not set for the upload to S3" ); + return; + } + bool rawDataAvailableToSend = false; + bool vehicleDataIsSet = false; + // Append signals with raw data to Ion file + for ( const auto &signal : triggeredVisionSystemData->signals ) + { + if ( signal.value.type != SignalType::COMPLEX_SIGNAL ) + { + continue; + } + rawDataAvailableToSend = true; + if ( !vehicleDataIsSet ) + { + vehicleDataIsSet = true; + // Setup the next Ion file data only once + mIonWriter->setupVehicleData( triggeredVisionSystemData ); + } + mIonWriter->append( signal ); + } + if ( !rawDataAvailableToSend ) + { + return; + } + + auto s3UploadMetadata = + getS3UploadMetadataForCollectionScheme( triggeredVisionSystemData->metadata.collectionSchemeID ); + if ( s3UploadMetadata == S3UploadMetadata() ) + { + FWE_LOG_WARN( "Collection scheme " + triggeredVisionSystemData->metadata.collectionSchemeID + + " no longer active" ); + return; + } + + // Get stream for the Ion file and upload it with S3 sender + auto streambufBuilder = mIonWriter->getStreambufBuilder(); + + std::string objectKey = s3UploadMetadata.prefix + std::to_string( triggeredVisionSystemData->eventID ) + "-" + + std::to_string( triggeredVisionSystemData->triggerTime ) + &DEFAULT_KEY_SUFFIX[0]; + auto resultCallback = [this, objectKey, triggeredVisionSystemData, callback]( + ConnectivityError result, std::shared_ptr data ) -> void { + if ( result != ConnectivityError::Success ) + { + std::shared_ptr dataToPersist; + if ( triggeredVisionSystemData->metadata.persist && data != nullptr ) + { + dataToPersist = std::make_shared( objectKey, S3UploadParams(), data ); + } + callback( false, dataToPersist ); + return; + } + + auto collectedData = std::make_shared(); + collectedData->metadata = triggeredVisionSystemData->metadata; + collectedData->eventID = triggeredVisionSystemData->eventID; + collectedData->triggerTime = triggeredVisionSystemData->triggerTime; + collectedData->uploadedS3Objects.push_back( UploadedS3Object{ objectKey, UploadedS3ObjectDataFormat::Cdr } ); + if ( !mUploadedS3Objects->push( std::move( collectedData ) ) ) + { + FWE_LOG_ERROR( "Collected data output buffer is full" ); + } + callback( true, nullptr ); + }; + mS3Sender->sendStream( std::move( streambufBuilder ), s3UploadMetadata, objectKey, resultCallback ); +} + +void +VisionSystemDataSender::processPersistedData( std::istream &data, + const Json::Value &metadata, + OnPersistedDataProcessedCallback callback ) +{ + static_cast( data ); + static_cast( metadata ); + static_cast( callback ); + + FWE_LOG_WARN( "Upload of persisted data is not supported for VisionSystemData" ); +} + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/VisionSystemDataSender.h b/src/VisionSystemDataSender.h new file mode 100644 index 00000000..43f710d0 --- /dev/null +++ b/src/VisionSystemDataSender.h @@ -0,0 +1,96 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "CollectionInspectionAPITypes.h" +#include "DataSenderIonWriter.h" +#include "DataSenderTypes.h" +#include "ICollectionScheme.h" +#include "ICollectionSchemeList.h" +#include "S3Sender.h" +#include +#include +#include +#include +#include +#include + +namespace Aws +{ +namespace IoTFleetWise +{ + +constexpr char DEFAULT_KEY_SUFFIX[] = ".10n"; // Ion is the only supported format + +/** + * @brief Struct that specifies the persistence and transmission attributes + * for the S3 upload + */ +struct S3UploadParams +{ + std::string region; // bucket region, set on the campaign level, attribute of S3 client + std::string bucketName; // bucket name, set on the campaign level, attribute of S3 request + std::string bucketOwner; // bucket owner account ID, set on the campaign level, attribute of S3 request + std::string objectName; // object key, attribute of S3 request + std::string uploadID; // upload ID of the multipart upload + uint16_t multipartID{ 0 }; // multipartID of a single part of the multipart upload, allowed values are 1 to 10000 + +public: + bool + operator==( const S3UploadParams &other ) const + { + return ( bucketName == other.bucketName ) && ( bucketOwner == other.bucketOwner ) && + ( objectName == other.objectName ) && ( region == other.region ) && ( uploadID == other.uploadID ) && + ( multipartID == other.multipartID ); + } + + bool + operator!=( const S3UploadParams &other ) const + { + return !( *this == other ); + } +}; + +class VisionSystemDataSender : public DataSender +{ + +public: + VisionSystemDataSender( std::shared_ptr uploadedS3Objects, + std::shared_ptr s3Sender, + std::shared_ptr ionWriter, + std::string vehicleName ); + + ~VisionSystemDataSender() override = default; + + void processData( std::shared_ptr data, OnDataProcessedCallback callback ) override; + + void processPersistedData( std::istream &data, + const Json::Value &metadata, + OnPersistedDataProcessedCallback callback ) override; + + virtual void onChangeCollectionSchemeList( + const std::shared_ptr &activeCollectionSchemes ); + +private: + std::shared_ptr mUploadedS3Objects; + std::shared_ptr mIonWriter; + std::shared_ptr mS3Sender; // might be nullptr + std::string mVehicleName; + std::mutex mActiveCollectionSchemeMutex; + std::shared_ptr mActiveCollectionSchemes; + + /** + * @brief Put vision system data into ion in chunks. Initiate serialization, compression, and + * upload for each partition. + * @param triggeredVisionSystemData collected data + * @param callback callback function to be called after data is processed + */ + void transformVisionSystemDataToIon( std::shared_ptr triggeredVisionSystemData, + OnDataProcessedCallback callback ); + + S3UploadMetadata getS3UploadMetadataForCollectionScheme( const std::string &collectionSchemeID ); +}; + +} // namespace IoTFleetWise +} // namespace Aws diff --git a/src/android_shared_library.cpp b/src/android_shared_library.cpp index 82bb7df6..824c65da 100644 --- a/src/android_shared_library.cpp +++ b/src/android_shared_library.cpp @@ -5,9 +5,11 @@ #include "IoTFleetWiseEngine.h" #include "IoTFleetWiseVersion.h" #include "LogLevel.h" +#include "SignalTypes.h" #include #include #include +#include #include #include #include @@ -134,7 +136,7 @@ Java_com_aws_iotfleetwise_Fwe_run( JNIEnv *env, mEngine = std::make_shared(); // Connect the Engine - if ( mEngine->connect( config ) && mEngine->start() ) + if ( mEngine->connect( config, boost::filesystem::path( "" ) ) && mEngine->start() ) { LOGI( std::string( "Started successfully" ) ); } @@ -228,7 +230,7 @@ Java_com_aws_iotfleetwise_Fwe_getVehiclePropertyInfo( JNIEnv *env, jobject me ) } extern "C" JNIEXPORT void JNICALL -Java_com_aws_iotfleetwise_Fwe_setVehicleProperty( JNIEnv *env, jobject me, jint signalId, jdouble value ) +Java_com_aws_iotfleetwise_Fwe_setVehicleProperty( JNIEnv *env, jobject me, jint signalId, jobject value ) { static_cast( env ); static_cast( me ); @@ -236,7 +238,30 @@ Java_com_aws_iotfleetwise_Fwe_setVehicleProperty( JNIEnv *env, jobject me, jint { return; } - mEngine->setVehicleProperty( static_cast( signalId ), value ); + + jclass doubleJClass = env->FindClass( "java/lang/Double" ); + jclass longJClass = env->FindClass( "java/lang/Long" ); + + if ( env->IsInstanceOf( value, doubleJClass ) ) + { + jmethodID doubleJMethodIdDoubleValue = env->GetMethodID( doubleJClass, "doubleValue", "()D" ); + jdouble doubleValue = env->CallDoubleMethod( value, doubleJMethodIdDoubleValue ); + mEngine->setVehicleProperty( + static_cast( signalId ), + Aws::IoTFleetWise::DecodedSignalValue( doubleValue, Aws::IoTFleetWise::SignalType::DOUBLE ) ); + } + else if ( env->IsInstanceOf( value, longJClass ) ) + { + jmethodID longJMethodIdLongValue = env->GetMethodID( longJClass, "longValue", "()J" ); + jlong longValue = env->CallLongMethod( value, longJMethodIdLongValue ); + mEngine->setVehicleProperty( + static_cast( signalId ), + Aws::IoTFleetWise::DecodedSignalValue( longValue, Aws::IoTFleetWise::SignalType::INT64 ) ); + } + else + { + LOGE( std::string( "Unsupported value type" ) ); + } } #endif diff --git a/src/main.cpp b/src/main.cpp index e4e3f1d8..8df3ab5c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,6 +6,7 @@ #include "IoTFleetWiseEngine.h" #include "IoTFleetWiseVersion.h" #include "LogLevel.h" +#include #include #include #include @@ -111,9 +112,11 @@ main( int argc, char *argv[] ) // Set system wide log level configureLogging( config ); + auto configFileDirectoryPath = boost::filesystem::absolute( configFilename ).parent_path(); + Aws::IoTFleetWise::IoTFleetWiseEngine engine; // Connect the Engine - if ( engine.connect( config ) && engine.start() ) + if ( engine.connect( config, configFileDirectoryPath ) && engine.start() ) { std::cout << "Started successfully" << std::endl; } diff --git a/test/unit/AaosVhalSourceTest.cpp b/test/unit/AaosVhalSourceTest.cpp index d56e9dff..827a8e12 100644 --- a/test/unit/AaosVhalSourceTest.cpp +++ b/test/unit/AaosVhalSourceTest.cpp @@ -5,6 +5,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "VehicleDataSourceTypes.h" #include "WaitUntil.h" @@ -32,6 +33,7 @@ class AaosVhalSourceTest : public ::testing::Test decoderMethod.format.mMessageID = 12345; CANSignalFormat sig1; CANSignalFormat sig2; + CANSignalFormat sig3; sig1.mOffset = 0x0207; sig1.mFirstBitPosition = 0xAA; sig1.mSizeInBits = 0x55; @@ -40,8 +42,13 @@ class AaosVhalSourceTest : public ::testing::Test sig2.mFirstBitPosition = 0x55; sig2.mSizeInBits = 0xAA; sig2.mSignalID = 0x5678; + sig3.mOffset = 0x020A; + sig3.mFirstBitPosition = 0x77; + sig3.mSizeInBits = 0xBB; + sig3.mSignalID = 0x8888; decoderMethod.format.mSignals.push_back( sig1 ); decoderMethod.format.mSignals.push_back( sig2 ); + decoderMethod.format.mSignals.push_back( sig3 ); frameMap[1] = decoderMethod; mDictionary = std::make_shared(); mDictionary->canMessageDecoderMethod[1] = frameMap; @@ -57,35 +64,49 @@ class AaosVhalSourceTest : public ::testing::Test TEST_F( AaosVhalSourceTest, testDecoding ) // NOLINT { - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); - AaosVhalSource vhalSource( signalBufferPtr ); + auto signalBuffer = std::make_shared( 100, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + AaosVhalSource vhalSource( signalBufferDistributor ); ASSERT_FALSE( vhalSource.init( INVALID_CAN_SOURCE_NUMERIC_ID, 1 ) ); ASSERT_TRUE( vhalSource.init( 1, 1 ) ); vhalSource.start(); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); vhalSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); - auto propInfo = vhalSource.getVehiclePropertyInfo(); - ASSERT_EQ( 2, propInfo.size() ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); + + std::unordered_map> propInfoBySignalId; + for ( auto propInfo : vhalSource.getVehiclePropertyInfo() ) + { + propInfoBySignalId[propInfo[3]] = propInfo; + } + ASSERT_EQ( 3, vhalSource.getVehiclePropertyInfo().size() ); + ASSERT_EQ( 3, propInfoBySignalId.size() ); auto sig1Info = std::array{ 0x0207, 0xAA, 0x55, 0x1234 }; - ASSERT_EQ( sig1Info, propInfo[0] ); + ASSERT_EQ( sig1Info, propInfoBySignalId[0x1234] ); auto sig2Info = std::array{ 0x0209, 0x55, 0xAA, 0x5678 }; - ASSERT_EQ( sig2Info, propInfo[1] ); - vhalSource.setVehicleProperty( 0x1234, 52.5761 ); - vhalSource.setVehicleProperty( 0x5678, 12.5761 ); + ASSERT_EQ( sig2Info, propInfoBySignalId[0x5678] ); + auto sig3Info = std::array{ 0x020A, 0x77, 0xBB, 0x8888 }; + ASSERT_EQ( sig3Info, propInfoBySignalId[0x8888] ); - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + vhalSource.setVehicleProperty( 0x1234, DecodedSignalValue( 52.5761, SignalType::DOUBLE ) ); + vhalSource.setVehicleProperty( 0x5678, DecodedSignalValue( 12, SignalType::DOUBLE ) ); + vhalSource.setVehicleProperty( 0x8888, DecodedSignalValue( 123456, SignalType::INT64 ) ); + + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto firstSignal = collectedDataFrame.mCollectedSignals[0]; - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto secondSignal = collectedDataFrame.mCollectedSignals[0]; + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); + auto thirdSignal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( firstSignal.signalID, 0x1234 ); ASSERT_EQ( secondSignal.signalID, 0x5678 ); + ASSERT_EQ( thirdSignal.signalID, 0x8888 ); ASSERT_NEAR( firstSignal.value.value.doubleVal, 52.5761, 0.0001 ); - ASSERT_NEAR( secondSignal.value.value.doubleVal, 12.5761, 0.0001 ); - - ASSERT_TRUE( vhalSource.stop() ); + ASSERT_EQ( secondSignal.value.value.doubleVal, 12 ); + ASSERT_EQ( thirdSignal.value.value.doubleVal, 123456 ); } } // namespace IoTFleetWise diff --git a/test/unit/AwsIotConnectivityModuleTest.cpp b/test/unit/AwsIotConnectivityModuleTest.cpp index e40ac5cf..5ea54381 100644 --- a/test/unit/AwsIotConnectivityModuleTest.cpp +++ b/test/unit/AwsIotConnectivityModuleTest.cpp @@ -3,32 +3,27 @@ #include "AwsIotConnectivityModule.h" #include "AwsBootstrap.h" -#include "AwsIotChannel.h" +#include "AwsIotReceiver.h" +#include "AwsIotSender.h" #include "AwsSDKMemoryManager.h" -#include "CacheAndPersist.h" #include "Clock.h" #include "ClockHandler.h" #include "IConnectionTypes.h" -#include "IConnectivityChannel.h" #include "IReceiver.h" -#include "ISender.h" #include "MqttClientWrapper.h" #include "MqttClientWrapperMock.h" -#include "PayloadManager.h" #include "WaitUntil.h" -#include #include #include #include +#include #include +#include #include #include #include #include -#include -#include #include -#include #include #include #include @@ -37,7 +32,6 @@ #include #include #include -#include #include #include @@ -52,6 +46,7 @@ using ::testing::AnyNumber; using ::testing::AtLeast; using ::testing::DoAll; using ::testing::Invoke; +using ::testing::MockFunction; using ::testing::Return; using ::testing::ReturnRef; using ::testing::SaveArg; @@ -73,9 +68,25 @@ class AwsIotConnectivityModuleTest : public ::testing::Test // Need to initialize the SDK to get proper error strings AwsBootstrap::getInstance().getClientBootStrap(); + mNegotiatedSettings.maximum_qos = AWS_MQTT5_QOS_AT_LEAST_ONCE; + mNegotiatedSettings.session_expiry_interval = 0; + mNegotiatedSettings.receive_maximum_from_server = 10000; + mNegotiatedSettings.maximum_packet_size_to_server = 10000; + mNegotiatedSettings.topic_alias_maximum_to_server = 50; + mNegotiatedSettings.topic_alias_maximum_to_client = 80; + mNegotiatedSettings.server_keep_alive = 60; + mNegotiatedSettings.retain_available = false; + mNegotiatedSettings.wildcard_subscriptions_available = false; + mNegotiatedSettings.subscription_identifiers_available = false; + mNegotiatedSettings.shared_subscriptions_available = false; + mNegotiatedSettings.rejoined_session = false; + std::string clientId = "client-1234"; + auto clientIdByteCursor = Aws::Crt::ByteCursorFromCString( clientId.c_str() ); + aws_mqtt5_negotiated_settings_init( Aws::Crt::ApiAllocator(), &mNegotiatedSettings, &clientIdByteCursor ); + mMqttClientWrapperMock = std::make_shared>(); - // We need to pass the client shared_ptr to AwsIotChannel as a reference, so we can't pass the pointer to the - // subclass (i.e. MqttClientWrapperMock). + // We need to pass the client shared_ptr to AwsIotSender and AwsIotReceiver as a reference, so we can't pass the + // pointer to the subclass (i.e. MqttClientWrapperMock). mMqttClientWrapper = mMqttClientWrapperMock; EXPECT_CALL( *mMqttClientWrapperMock, MockedOperatorBool() ) .Times( AnyNumber() ) @@ -85,11 +96,30 @@ class AwsIotConnectivityModuleTest : public ::testing::Test mMqttClientBuilderWrapperMock->mOnConnectionSuccessCallback( eventData ); return true; } ) ); - ON_CALL( *mMqttClientWrapperMock, Stop() ).WillByDefault( Invoke( [this]() noexcept -> bool { - Aws::Crt::Mqtt5::OnStoppedEventData eventData; - mMqttClientBuilderWrapperMock->mOnStoppedCallback( eventData ); - return true; - } ) ); + ON_CALL( *mMqttClientWrapperMock, Stop( _ ) ) + .WillByDefault( Invoke( + [this]( std::shared_ptr disconnectOptions ) noexcept -> bool { + static_cast( disconnectOptions ); + Aws::Crt::Mqtt5::OnStoppedEventData eventData; + mMqttClientBuilderWrapperMock->mOnStoppedCallback( eventData ); + return true; + } ) ); + ON_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ) + .WillByDefault( + Invoke( [this]( std::shared_ptr, + Aws::Crt::Mqtt5::OnSubscribeCompletionHandler onSubscribeCompletionCallback ) -> bool { + onSubscribeCompletionCallback( AWS_ERROR_SUCCESS, nullptr ); + mSubscribeCount++; + return true; + } ) ); + ON_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ) + .WillByDefault( Invoke( + [this]( + std::shared_ptr, + Aws::Crt::Mqtt5::OnUnsubscribeCompletionHandler onUnsubscribeCompletionCallback ) noexcept -> bool { + onUnsubscribeCompletionCallback( AWS_ERROR_SUCCESS, nullptr ); + return true; + } ) ); mMqttClientBuilderWrapperMock = std::make_shared>(); ON_CALL( *mMqttClientBuilderWrapperMock, Build() ).WillByDefault( Return( mMqttClientWrapperMock ) ); @@ -97,6 +127,8 @@ class AwsIotConnectivityModuleTest : public ::testing::Test .WillByDefault( ReturnRef( *mMqttClientBuilderWrapperMock ) ); ON_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ) .WillByDefault( DoAll( SaveArg<0>( &mConnectPacket ), ReturnRef( *mMqttClientBuilderWrapperMock ) ) ); + ON_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ) + .WillByDefault( ReturnRef( *mMqttClientBuilderWrapperMock ) ); ON_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ) .WillByDefault( ReturnRef( *mMqttClientBuilderWrapperMock ) ); ON_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ) @@ -108,11 +140,34 @@ class AwsIotConnectivityModuleTest : public ::testing::Test std::make_shared( "", "clientIdTest", mMqttClientBuilderWrapperMock ); } + void + TearDown() override + { + aws_mqtt5_negotiated_settings_clean_up( &mNegotiatedSettings ); + } + + void + publishToTopic( const std::string &topic, const std::string &data ) + { + { + Aws::Crt::Mqtt5::PublishReceivedEventData eventData; + + auto publishPacket = + std::make_shared( topic.c_str(), + Aws::Crt::ByteCursorFromCString( data.c_str() ), + Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); + eventData.publishPacket = publishPacket; + mMqttClientBuilderWrapperMock->mOnPublishReceivedHandlerCallback( eventData ); + } + } + std::shared_ptr> mMqttClientWrapperMock; std::shared_ptr mMqttClientWrapper; std::shared_ptr> mMqttClientBuilderWrapperMock; std::shared_ptr mConnectivityModule; std::shared_ptr mConnectPacket; + aws_mqtt5_negotiated_settings mNegotiatedSettings; + std::atomic mSubscribeCount{ 0 }; }; /** @@ -127,16 +182,17 @@ class AwsIotConnectivityModuleTestAfterSuccessfulConnection : public AwsIotConne { AwsIotConnectivityModuleTest::SetUp(); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, - WithClientExtendedValidationAndFlowControl( - Aws::Crt::Mqtt5::ClientExtendedValidationAndFlowControl::AWS_MQTT5_EVAFCO_NONE ) ) - .Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, - WithSessionBehavior( Aws::Crt::Mqtt5::ClientSessionBehaviorType::AWS_MQTT5_CSBT_REJOIN_POST_SUCCESS ) ) + WithClientExtendedValidationAndFlowControl( + Aws::Crt::Mqtt5::ClientExtendedValidationAndFlowControl::AWS_MQTT5_EVAFCO_AWS_IOT_CORE_DEFAULTS ) ) + .Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, + WithOfflineQueueBehavior( + Aws::Crt::Mqtt5::ClientOperationQueueBehaviorType::AWS_MQTT5_COQBT_FAIL_ALL_ON_DISCONNECT ) ) .Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( 3000 ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( MQTT_PING_TIMEOUT_MS ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); @@ -150,7 +206,7 @@ class AwsIotConnectivityModuleTestAfterSuccessfulConnection : public AwsIotConne { AwsIotConnectivityModuleTest::TearDown(); // This should be called on AwsIotConnectivityModule destruction - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); } }; @@ -174,16 +230,18 @@ TEST_F( AwsIotConnectivityModuleTest, connectSuccessfully ) { EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( MQTT_PING_TIMEOUT_MS ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); ASSERT_TRUE( mConnectivityModule->connect() ); + ASSERT_EQ( mConnectPacket->getKeepAliveIntervalSec(), MQTT_KEEP_ALIVE_INTERVAL_SECONDS ); + ASSERT_EQ( mConnectPacket->getSessionExpiryIntervalSec().value(), MQTT_SESSION_EXPIRY_INTERVAL_SECONDS ); WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); ASSERT_TRUE( mConnectivityModule->disconnect() ); } @@ -196,7 +254,7 @@ TEST_F( AwsIotConnectivityModuleTest, connectSuccessfullyWithRootCA ) EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithCertificateAuthority( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); @@ -206,7 +264,57 @@ TEST_F( AwsIotConnectivityModuleTest, connectSuccessfullyWithRootCA ) WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); +} + +TEST_F( AwsIotConnectivityModuleTest, connectSuccessfullyWithOverridenConnectionConfig ) +{ + AwsIotConnectivityConfig mqttConnectionConfig; + mqttConnectionConfig.keepAliveIntervalSeconds = 321; + mqttConnectionConfig.pingTimeoutMs = 17; + + mConnectivityModule = std::make_shared( + "", "clientIdTest", mMqttClientBuilderWrapperMock, mqttConnectionConfig ); + + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( 17 ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); + + ASSERT_TRUE( mConnectivityModule->connect() ); + ASSERT_EQ( mConnectPacket->getKeepAliveIntervalSec(), 321 ); + ASSERT_EQ( mConnectPacket->getSessionExpiryIntervalSec().value(), 0 ); + + WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); +} + +TEST_F( AwsIotConnectivityModuleTest, connectSuccessfullyWithPersistentSession ) +{ + AwsIotConnectivityConfig mqttConnectionConfig; + mqttConnectionConfig.sessionExpiryIntervalSeconds = 7890; + + mConnectivityModule = std::make_shared( + "", "clientIdTest", mMqttClientBuilderWrapperMock, mqttConnectionConfig ); + + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( AWS_MQTT5_CSBT_REJOIN_ALWAYS ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( MQTT_PING_TIMEOUT_MS ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); + + ASSERT_TRUE( mConnectivityModule->connect() ); + ASSERT_EQ( mConnectPacket->getKeepAliveIntervalSec(), MQTT_KEEP_ALIVE_INTERVAL_SECONDS ); + ASSERT_EQ( mConnectPacket->getSessionExpiryIntervalSec().value(), 7890 ); + + WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); } /** @brief Test trying to connect, where creation of the client fails */ @@ -214,7 +322,7 @@ TEST_F( AwsIotConnectivityModuleTest, connectFailsOnClientCreation ) { EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); EXPECT_CALL( *mMqttClientWrapperMock, MockedOperatorBool() ) @@ -230,7 +338,7 @@ TEST_F( AwsIotConnectivityModuleTest, connectionInterrupted ) { EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); @@ -239,7 +347,7 @@ TEST_F( AwsIotConnectivityModuleTest, connectionInterrupted ) WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); { Aws::Crt::Mqtt5::OnDisconnectionEventData eventData; @@ -261,7 +369,7 @@ TEST_F( AwsIotConnectivityModuleTest, clientFailsToStartFirstAttempt ) { EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); { @@ -278,7 +386,7 @@ TEST_F( AwsIotConnectivityModuleTest, clientFailsToStartFirstAttempt ) } ) ); } EXPECT_CALL( *mMqttClientWrapperMock, LastError() ).Times( 1 ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); ASSERT_TRUE( mConnectivityModule->connect() ); @@ -314,13 +422,13 @@ TEST_F( AwsIotConnectivityModuleTest, connectFailsServerUnavailableWithDelay ) EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); // Don't automatically invoke the callback, we want to call it manually. EXPECT_CALL( *mMqttClientWrapperMock, Start() ).WillOnce( Return( true ) ); // We want to see exactly one call to disconnect - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); ASSERT_TRUE( mConnectivityModule->connect() ); std::this_thread::sleep_for( std::chrono::milliseconds( 20 ) ); @@ -331,118 +439,295 @@ TEST_F( AwsIotConnectivityModuleTest, connectFailsServerUnavailableWithDelay ) ASSERT_FALSE( mConnectivityModule->isAlive() ); // Don't expect more calls on destruction since it is already disconnected - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 0 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 0 ); } /** @brief Test subscribing without a configured topic, expect an error */ TEST_F( AwsIotConnectivityModuleTest, subscribeWithoutTopic ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "", false ); - ASSERT_EQ( channel.subscribe(), ConnectivityError::NotConfigured ); - channel.invalidateConnection(); + AwsIotReceiver receiver( mConnectivityModule.get(), mMqttClientWrapper, "" ); + ASSERT_EQ( receiver.subscribe(), ConnectivityError::NotConfigured ); + receiver.invalidateConnection(); } /** @brief Test subscribing without being connected, expect an error */ TEST_F( AwsIotConnectivityModuleTest, subscribeWithoutBeingConnected ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); - ASSERT_EQ( channel.subscribe(), ConnectivityError::NoConnection ); - channel.invalidateConnection(); + AwsIotReceiver receiver( mConnectivityModule.get(), mMqttClientWrapper, "topic" ); + ASSERT_EQ( receiver.subscribe(), ConnectivityError::NoConnection ); + receiver.invalidateConnection(); } TEST_F( AwsIotConnectivityModuleTest, receiveMessage ) { - std::vector> receivedDataChannel1; - std::vector> receivedDataChannel2; + std::vector> receivedDataReceiver1; + std::vector> receivedDataReceiver2; + std::vector> receivedDataReceiver3; - auto channel1 = mConnectivityModule->createNewChannel( nullptr, "topic1", true ); - auto channel2 = mConnectivityModule->createNewChannel( nullptr, "topic2", true ); + auto receiver1 = mConnectivityModule->createReceiver( "topic1" ); + auto receiver2 = mConnectivityModule->createReceiver( "topic2" ); + auto receiver3 = mConnectivityModule->createReceiver( "topic3/+/request" ); - channel1->subscribeToDataReceived( [&]( const ReceivedChannelMessage &message ) { - receivedDataChannel1.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + receiver1->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver1.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + receiver2->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver2.emplace_back( std::string( message.buf, message.buf + message.size ), message ); } ); - channel2->subscribeToDataReceived( [&]( const ReceivedChannelMessage &message ) { - receivedDataChannel2.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + receiver3->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver3.emplace_back( std::string( message.buf, message.buf + message.size ), message ); } ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); - EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ) - .Times( 2 ) - .WillRepeatedly( - Invoke( [this]( std::shared_ptr, - Aws::Crt::Mqtt5::OnSubscribeCompletionHandler onSubscribeCompletionCallback ) -> bool { - onSubscribeCompletionCallback( AWS_ERROR_SUCCESS, nullptr ); - return true; - } ) ); + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 3 ); ASSERT_TRUE( mConnectivityModule->connect() ); WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); - WAIT_ASSERT_TRUE( static_cast( channel1.get() )->isAlive() ); - WAIT_ASSERT_TRUE( static_cast( channel2.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver2.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver3.get() )->isAlive() ); + auto publishTime = ClockHandler::getClock()->monotonicTimeSinceEpochMs(); // Simulate messages coming from MQTT client - std::string data1 = "data1"; - uint64_t expectedMessageExpiryMonotonicTimeSinceEpochMs = UINT64_MAX; + publishToTopic( "topic1", "data1" ); + publishToTopic( "topic2", "data2" ); + publishToTopic( "topic3/12345/request", "data3" ); + + ASSERT_EQ( receivedDataReceiver1.size(), 1 ); + ASSERT_EQ( receivedDataReceiver1[0].first, "data1" ); + ASSERT_GE( receivedDataReceiver1[0].second.receivedMonotonicTimeMs, publishTime ); + // Give some margin for error due to test being slow, but make sure that timeout is not much more + // than expected. + ASSERT_LE( receivedDataReceiver1[0].second.receivedMonotonicTimeMs, publishTime + 500 ); + + ASSERT_EQ( receivedDataReceiver2.size(), 1 ); + ASSERT_EQ( receivedDataReceiver2[0].first, "data2" ); + + ASSERT_EQ( receivedDataReceiver3.size(), 1 ); + ASSERT_EQ( receivedDataReceiver3[0].first, "data3" ); + + // Should be called on destruction + EXPECT_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ).Times( 3 ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); + + ASSERT_TRUE( mConnectivityModule->disconnect() ); +} + +TEST_F( AwsIotConnectivityModuleTest, retryFailedSubscription ) +{ + std::vector> receivedDataReceiver1; + std::vector> receivedDataReceiver2; + + auto receiver1 = mConnectivityModule->createReceiver( "topic1" ); + auto receiver2 = mConnectivityModule->createReceiver( "topic2" ); + + receiver1->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver1.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + receiver2->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver2.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); + { - Aws::Crt::Mqtt5::PublishReceivedEventData eventData; - - auto publishPacket = - std::make_shared( "topic1", - Aws::Crt::ByteCursorFromCString( data1.c_str() ), - Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); - auto messageExpiryIntervalSec = 5; - publishPacket->WithMessageExpiryIntervalSec( messageExpiryIntervalSec ); - eventData.publishPacket = publishPacket; - expectedMessageExpiryMonotonicTimeSinceEpochMs = - ClockHandler::getClock()->monotonicTimeSinceEpochMs() + ( messageExpiryIntervalSec * 1000 ); - mMqttClientBuilderWrapperMock->mOnPublishReceivedHandlerCallback( eventData ); + Sequence seq; + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 1 ).InSequence( seq ); + // One of the subscriptions will fail + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ) + .InSequence( seq ) + .WillOnce( + Invoke( [this]( std::shared_ptr, + Aws::Crt::Mqtt5::OnSubscribeCompletionHandler onSubscribeCompletionCallback ) -> bool { + onSubscribeCompletionCallback( AWS_ERROR_MQTT_TIMEOUT, nullptr ); + return true; + } ) ); + // Then on retry, we should expect only one additional Subscribe call (the topic that succeeded + // should not be re-subscribed) + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 1 ).InSequence( seq ); } - std::string data2 = "data2"; + + ASSERT_TRUE( mConnectivityModule->connect() ); + + WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver2.get() )->isAlive() ); + + // Simulate messages coming from MQTT client + publishToTopic( "topic1", "data1" ); + publishToTopic( "topic2", "data2" ); + + ASSERT_EQ( receivedDataReceiver1.size(), 1 ); + ASSERT_EQ( receivedDataReceiver1[0].first, "data1" ); + + ASSERT_EQ( receivedDataReceiver2.size(), 1 ); + ASSERT_EQ( receivedDataReceiver2[0].first, "data2" ); + + // Should be called on destruction + EXPECT_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ).Times( 2 ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); + + ASSERT_TRUE( mConnectivityModule->disconnect() ); +} + +TEST_F( AwsIotConnectivityModuleTest, resubscribeToAllTopicsWhenNotRejoiningExistingSession ) +{ + std::vector> receivedDataReceiver1; + std::vector> receivedDataReceiver2; + + auto receiver1 = mConnectivityModule->createReceiver( "topic1" ); + auto receiver2 = mConnectivityModule->createReceiver( "topic2" ); + + receiver1->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver1.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + receiver2->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver2.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 2 ); + + ASSERT_TRUE( mConnectivityModule->connect() ); + + WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); + WAIT_ASSERT_EQ( mSubscribeCount.load(), 2 ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver2.get() )->isAlive() ); + + // Simulate a reconnection. Note that when using the real client, once we start the client, + // both failure and success callbacks will be called without requiring any action from our side. + mMqttClientBuilderWrapperMock->mOnConnectionFailureCallback( Aws::Crt::Mqtt5::OnConnectionFailureEventData() ); + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 2 ); { - Aws::Crt::Mqtt5::PublishReceivedEventData eventData; - auto publishPacket = - std::make_shared( "topic2", - Aws::Crt::ByteCursorFromCString( data2.c_str() ), - Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); - eventData.publishPacket = publishPacket; - mMqttClientBuilderWrapperMock->mOnPublishReceivedHandlerCallback( eventData ); + mNegotiatedSettings.rejoined_session = false; + Aws::Crt::Mqtt5::OnConnectionSuccessEventData eventData; + eventData.negotiatedSettings = std::make_shared( mNegotiatedSettings ); + mMqttClientBuilderWrapperMock->mOnConnectionSuccessCallback( eventData ); } + WAIT_ASSERT_EQ( mSubscribeCount.load(), 4 ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver2.get() )->isAlive() ); - ASSERT_EQ( receivedDataChannel1.size(), 1 ); - ASSERT_EQ( receivedDataChannel1[0].first, "data1" ); - ASSERT_GE( receivedDataChannel1[0].second.messageExpiryMonotonicTimeSinceEpochMs, - expectedMessageExpiryMonotonicTimeSinceEpochMs ); - // Give some margin for error due to test being slow, but make sure that timeout is not much more - // than expected. - ASSERT_LE( receivedDataChannel1[0].second.messageExpiryMonotonicTimeSinceEpochMs, - expectedMessageExpiryMonotonicTimeSinceEpochMs + 500 ); + // Simulate messages coming from MQTT client + publishToTopic( "topic1", "data1" ); + publishToTopic( "topic2", "data2" ); + + ASSERT_EQ( receivedDataReceiver1.size(), 1 ); + ASSERT_EQ( receivedDataReceiver1[0].first, "data1" ); - ASSERT_EQ( receivedDataChannel2.size(), 1 ); - ASSERT_EQ( receivedDataChannel2[0].first, "data2" ); - ASSERT_EQ( receivedDataChannel2[0].second.messageExpiryMonotonicTimeSinceEpochMs, 0 ); + ASSERT_EQ( receivedDataReceiver2.size(), 1 ); + ASSERT_EQ( receivedDataReceiver2[0].first, "data2" ); // Should be called on destruction - EXPECT_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ) - .Times( 2 ) - .WillRepeatedly( Invoke( - [this]( std::shared_ptr, - Aws::Crt::Mqtt5::OnUnsubscribeCompletionHandler onUnsubscribeCompletionCallback ) noexcept -> bool { - onUnsubscribeCompletionCallback( AWS_ERROR_SUCCESS, nullptr ); + EXPECT_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ).Times( 2 ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); + + ASSERT_TRUE( mConnectivityModule->disconnect() ); +} + +TEST_F( AwsIotConnectivityModuleTest, resubscribeOnlyToFailedTopicsWhenRejoiningExistingSession ) +{ + std::vector> receivedDataReceiver1; + std::vector> receivedDataReceiver2; + + auto receiver1 = mConnectivityModule->createReceiver( "topic1" ); + auto receiver2 = mConnectivityModule->createReceiver( "topic2" ); + + receiver1->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver1.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + receiver2->subscribeToDataReceived( [&]( const ReceivedConnectivityMessage &message ) { + receivedDataReceiver2.emplace_back( std::string( message.buf, message.buf + message.size ), message ); + } ); + + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Start() ).Times( 1 ); + + // One of the subscriptions will fail + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ) + .Times( AnyNumber() ) + .WillRepeatedly( + Invoke( [this]( std::shared_ptr subscribePacket, + Aws::Crt::Mqtt5::OnSubscribeCompletionHandler onSubscribeCompletionCallback ) -> bool { + aws_mqtt5_packet_subscribe_view rawSubscribeOptions; + subscribePacket->initializeRawOptions( rawSubscribeOptions ); + if ( byteCursorToString( rawSubscribeOptions.subscriptions[0].topic_filter ) == "topic1" ) + { + onSubscribeCompletionCallback( AWS_ERROR_SUCCESS, nullptr ); + } + else + { + onSubscribeCompletionCallback( AWS_ERROR_MQTT_TIMEOUT, nullptr ); + } + mSubscribeCount++; return true; } ) ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + ASSERT_TRUE( mConnectivityModule->connect() ); + + WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); + WAIT_ASSERT_EQ( mSubscribeCount.load(), 2 ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + ASSERT_FALSE( static_cast( receiver2.get() )->isAlive() ); + + // Simulate a reconnection. Note that when using the real client, once we start the client, + // both failure and success callbacks will be called without requiring any action from our side. + mMqttClientBuilderWrapperMock->mOnConnectionFailureCallback( Aws::Crt::Mqtt5::OnConnectionFailureEventData() ); + EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ).Times( 1 ); + { + mNegotiatedSettings.rejoined_session = true; + Aws::Crt::Mqtt5::OnConnectionSuccessEventData eventData; + eventData.negotiatedSettings = std::make_shared( mNegotiatedSettings ); + mMqttClientBuilderWrapperMock->mOnConnectionSuccessCallback( eventData ); + } + + WAIT_ASSERT_EQ( mSubscribeCount.load(), 3 ); + WAIT_ASSERT_TRUE( static_cast( receiver1.get() )->isAlive() ); + WAIT_ASSERT_TRUE( static_cast( receiver2.get() )->isAlive() ); + + // Simulate messages coming from MQTT client + publishToTopic( "topic1", "data1" ); + publishToTopic( "topic2", "data2" ); + + ASSERT_EQ( receivedDataReceiver1.size(), 1 ); + ASSERT_EQ( receivedDataReceiver1[0].first, "data1" ); + + ASSERT_EQ( receivedDataReceiver2.size(), 1 ); + ASSERT_EQ( receivedDataReceiver2[0].first, "data2" ); + + // Should be called on destruction + EXPECT_CALL( *mMqttClientWrapperMock, Unsubscribe( _, _ ) ).Times( 2 ); + + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); ASSERT_TRUE( mConnectivityModule->disconnect() ); } -TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, receiveMessageFromTopicWithNoChannel ) +TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, receiveMessageFromTopicWithNoReceiver ) { // Simulate messages coming from MQTT client std::string data1 = "data1"; @@ -458,7 +743,7 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, receiveMessageFro /** @brief Test successful subscription */ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, subscribeSuccessfully ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotReceiver receiver( mConnectivityModule.get(), mMqttClientWrapper, "topic" ); std::shared_ptr subscribeOptions; EXPECT_CALL( *mMqttClientWrapperMock, Subscribe( _, _ ) ) @@ -472,7 +757,7 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, subscribeSuccessf onSubscribeCompletionCallback( errorCode, nullptr ); return true; } ) ); - ASSERT_EQ( channel.subscribe(), ConnectivityError::Success ); + ASSERT_EQ( receiver.subscribe(), ConnectivityError::Success ); aws_mqtt5_packet_subscribe_view rawSubscribeOptions; subscribeOptions->initializeRawOptions( rawSubscribeOptions ); ASSERT_EQ( rawSubscribeOptions.subscription_count, 1 ); @@ -490,52 +775,64 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, subscribeSuccessf onUnsubscribeCompletionCallback( errorCode, nullptr ); return true; } ) ); - channel.unsubscribe(); + receiver.unsubscribe(); aws_mqtt5_packet_unsubscribe_view rawUnsubscribeOptions; unsubscribeOptions->initializeRawOptions( rawUnsubscribeOptions ); ASSERT_EQ( rawUnsubscribeOptions.topic_filter_count, 1 ); ASSERT_EQ( byteCursorToString( rawUnsubscribeOptions.topic_filters[0] ), "topic" ); - channel.invalidateConnection(); + receiver.invalidateConnection(); } /** @brief Test without a configured topic, expect an error */ TEST_F( AwsIotConnectivityModuleTest, sendWithoutTopic ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); std::uint8_t input[] = { 0xca, 0xfe }; - ASSERT_EQ( channel.sendBuffer( input, sizeof( input ) ), ConnectivityError::NotConfigured ); - channel.invalidateConnection(); + MockFunction resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::NotConfigured ) ).Times( 1 ); + sender.sendBuffer( input, sizeof( input ), resultCallback.AsStdFunction() ); + sender.invalidateConnection(); } /** @brief Test sending without a connection, expect an error */ TEST_F( AwsIotConnectivityModuleTest, sendWithoutConnection ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); + MockFunction resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::NoConnection ) ).Times( 1 ); std::uint8_t input[] = { 0xca, 0xfe }; - ASSERT_EQ( channel.sendBuffer( input, sizeof( input ) ), ConnectivityError::NoConnection ); - channel.invalidateConnection(); + sender.sendBuffer( input, sizeof( input ), resultCallback.AsStdFunction() ); + sender.invalidateConnection(); } /** @brief Test passing a null pointer, expect an error */ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendWrongInput ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); - ASSERT_EQ( channel.sendBuffer( nullptr, 10 ), ConnectivityError::WrongInputData ); - channel.invalidateConnection(); + MockFunction resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::WrongInputData ) ).Times( 1 ); + sender.sendBuffer( nullptr, 10, resultCallback.AsStdFunction() ); + sender.invalidateConnection(); } /** @brief Test sending a message larger then the maximum send size, expect an error */ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendTooBig ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); + MockFunction resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::WrongInputData ) ).Times( 1 ); std::vector a; - a.resize( channel.getMaxSendSize() + 1U ); - ASSERT_EQ( channel.sendBuffer( a.data(), a.size() ), ConnectivityError::WrongInputData ); - channel.invalidateConnection(); + a.resize( sender.getMaxSendSize() + 1U ); + sender.sendBuffer( a.data(), a.size(), resultCallback.AsStdFunction() ); + sender.invalidateConnection(); } /** @brief Test sending multiple messages. The API supports queuing of messages, so send more than one @@ -543,7 +840,8 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendTooBig ) * messages as failed to send to check that path. */ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendMultiple ) { - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); std::uint8_t input[] = { 0xca, 0xfe }; std::list completeHandlers; @@ -558,68 +856,58 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendMultiple ) } ) ); // Queue 2 packets - ASSERT_EQ( channel.sendBuffer( input, sizeof( input ) ), ConnectivityError::Success ); - ASSERT_EQ( channel.sendBuffer( input, sizeof( input ) ), ConnectivityError::Success ); + MockFunction resultCallback1; + sender.sendBuffer( input, sizeof( input ), resultCallback1.AsStdFunction() ); + MockFunction resultCallback2; + sender.sendBuffer( input, sizeof( input ), resultCallback2.AsStdFunction() ); // Confirm 1st + EXPECT_CALL( resultCallback1, Call( ConnectivityError::Success ) ).Times( 1 ); completeHandlers.front().operator()( 0, std::make_shared() ); completeHandlers.pop_front(); // Queue another: - ASSERT_EQ( channel.sendBuffer( input, sizeof( input ) ), ConnectivityError::Success ); + MockFunction resultCallback3; + sender.sendBuffer( input, sizeof( input ), resultCallback3.AsStdFunction() ); // Confirm 2nd + EXPECT_CALL( resultCallback2, Call( ConnectivityError::Success ) ).Times( 1 ); completeHandlers.front().operator()( 0, std::make_shared() ); completeHandlers.pop_front(); // Confirm 3rd (Not a test case failure, but a stimulated failure for code coverage) + EXPECT_CALL( resultCallback3, Call( ConnectivityError::TransmissionError ) ).Times( 1 ); completeHandlers.front().operator()( 1, std::make_shared( 1 ) ); completeHandlers.pop_front(); - ASSERT_EQ( channel.getPayloadCountSent(), 2 ); + ASSERT_EQ( sender.getPayloadCountSent(), 2 ); - channel.invalidateConnection(); + sender.invalidateConnection(); } -/** @brief Test SDK exceeds RAM and Channel stops sending */ +/** @brief Test SDK exceeds RAM and Sender stops sending */ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sdkRAMExceeded ) { auto &memMgr = AwsSDKMemoryManager::getInstance(); - void *alloc1 = memMgr.AllocateMemory( 600000000, alignof( std::size_t ) ); - ASSERT_NE( alloc1, nullptr ); - memMgr.FreeMemory( alloc1 ); - - void *alloc2 = memMgr.AllocateMemory( 600000010, alignof( std::size_t ) ); - ASSERT_NE( alloc2, nullptr ); - memMgr.FreeMemory( alloc2 ); - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); + AwsIotSender sender( + mConnectivityModule.get(), mMqttClientWrapper, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ); std::array input = { 0xCA, 0xFE }; const auto required = input.size() * sizeof( std::uint8_t ); { - void *alloc3 = - memMgr.AllocateMemory( 50 * AwsSDKMemoryManager::getInstance().getLimit(), alignof( std::size_t ) ); - ASSERT_NE( alloc3, nullptr ); + auto reservedMemory = AwsSDKMemoryManager::getInstance().getLimit(); + ASSERT_TRUE( memMgr.reserveMemory( reservedMemory ) ); - ASSERT_EQ( channel.sendBuffer( input.data(), input.size() * sizeof( std::uint8_t ) ), - ConnectivityError::QuotaReached ); - memMgr.FreeMemory( alloc3 ); + MockFunction resultCallback1; + EXPECT_CALL( resultCallback1, Call( ConnectivityError::QuotaReached ) ).Times( 1 ); + sender.sendBuffer( input.data(), input.size() * sizeof( std::uint8_t ), resultCallback1.AsStdFunction() ); + ASSERT_EQ( memMgr.releaseReservedMemory( reservedMemory ), 0 ); } { - constexpr auto offset = alignof( std::max_align_t ); - // check that we will be out of memory even if we allocate less than the max because of the allocator's offset - // in the below alloc we are leaving space for the input - auto alloc4 = memMgr.AllocateMemory( AwsSDKMemoryManager::getInstance().getLimit() - ( offset + required ), - alignof( std::size_t ) ); - ASSERT_NE( alloc4, nullptr ); - ASSERT_EQ( channel.sendBuffer( input.data(), sizeof( input ) ), ConnectivityError::QuotaReached ); - memMgr.FreeMemory( alloc4 ); - // check that allocation and hence send succeed when there is just enough memory // here we subtract the offset twice - once for MAXIMUM_AWS_SDK_HEAP_MEMORY_BYTES and once for the input - auto alloc5 = memMgr.AllocateMemory( - AwsSDKMemoryManager::getInstance().getLimit() - ( ( 2 * offset ) + required ), alignof( std::size_t ) ); - ASSERT_NE( alloc5, nullptr ); + auto reservedMemory = AwsSDKMemoryManager::getInstance().getLimit() - required; + ASSERT_TRUE( memMgr.reserveMemory( reservedMemory ) ); std::list completeHandlers; EXPECT_CALL( *mMqttClientWrapperMock, Publish( _, _ ) ) @@ -632,221 +920,18 @@ TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sdkRAMExceeded ) return true; } ) ); - ASSERT_EQ( channel.sendBuffer( input.data(), sizeof( input ) ), ConnectivityError::Success ); - memMgr.FreeMemory( alloc5 ); + MockFunction resultCallback3; + sender.sendBuffer( input.data(), sizeof( input ), resultCallback3.AsStdFunction() ); // // Confirm 1st + EXPECT_CALL( resultCallback3, Call( ConnectivityError::Success ) ).Times( 1 ); completeHandlers.front().operator()( 0, std::make_shared() ); completeHandlers.pop_front(); - } - - channel.invalidateConnection(); -} - -/** @brief Test sending file over MQTT without topic */ -TEST_F( AwsIotConnectivityModuleTest, sendFileOverMQTTNoTopic ) -{ - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "", false ); - std::string filename{ "testFile.json" }; - ASSERT_EQ( channel.sendFile( filename, 0 ), ConnectivityError::NotConfigured ); - channel.invalidateConnection(); -} - -/** @brief Test sending file over MQTT, payload manager not defined */ -TEST_F( AwsIotConnectivityModuleTest, sendFileOverMQTTNoPayloadManager ) -{ - AwsIotChannel channel( mConnectivityModule.get(), nullptr, mMqttClientWrapper, "topic", false ); - std::string filename{ "testFile.json" }; - ASSERT_EQ( channel.sendFile( filename, 0 ), ConnectivityError::NotConfigured ); - channel.invalidateConnection(); -} - -/** @brief Test sending file over MQTT, filename not defined */ -TEST_F( AwsIotConnectivityModuleTest, sendFileOverMQTTNoFilename ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; - } - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - std::string filename; - ASSERT_EQ( channel.sendFile( filename, 0 ), ConnectivityError::WrongInputData ); - channel.invalidateConnection(); -} - -/** @brief Test sending file over MQTT, file size too big */ -TEST_F( AwsIotConnectivityModuleTest, sendFileOverMQTTBigFile ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; - } - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - std::string filename = "testFile.json"; - ASSERT_EQ( channel.sendFile( filename, 150000 ), ConnectivityError::WrongInputData ); - channel.invalidateConnection(); -} - -TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendFileOverMQTTNoFile ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; - } - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - - std::string filename = "testFile.json"; - ASSERT_EQ( channel.sendFile( filename, 100 ), ConnectivityError::WrongInputData ); - - channel.invalidateConnection(); -} - -TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendFileOverMQTTSdkRAMExceeded ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; - } - - auto &memMgr = AwsSDKMemoryManager::getInstance(); - void *alloc1 = memMgr.AllocateMemory( 600000000, alignof( std::size_t ) ); - ASSERT_NE( alloc1, nullptr ); - memMgr.FreeMemory( alloc1 ); - - void *alloc2 = memMgr.AllocateMemory( 600000010, alignof( std::size_t ) ); - ASSERT_NE( alloc2, nullptr ); - memMgr.FreeMemory( alloc2 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - - // Fake file content - std::array input = { 0xCA, 0xFE }; - const auto required = input.size() * sizeof( std::uint8_t ); - std::string filename = "testFile.bin"; - { - void *alloc3 = - memMgr.AllocateMemory( 50 * AwsSDKMemoryManager::getInstance().getLimit(), alignof( std::size_t ) ); - ASSERT_NE( alloc3, nullptr ); - CollectionSchemeParams collectionSchemeParams; - collectionSchemeParams.persist = true; - collectionSchemeParams.compression = false; - ASSERT_EQ( channel.sendFile( filename, input.size() * sizeof( std::uint8_t ), collectionSchemeParams ), - ConnectivityError::QuotaReached ); - memMgr.FreeMemory( alloc3 ); - } - { - constexpr auto offset = alignof( std::max_align_t ); - // check that we will be out of memory even if we allocate less than the max because of the allocator's - // offset in the below alloc we are leaving space for the input - auto alloc4 = memMgr.AllocateMemory( AwsSDKMemoryManager::getInstance().getLimit() - ( offset + required ), - alignof( std::size_t ) ); - ASSERT_NE( alloc4, nullptr ); - ASSERT_EQ( channel.sendFile( filename, sizeof( input ) ), ConnectivityError::QuotaReached ); - memMgr.FreeMemory( alloc4 ); - } - - channel.invalidateConnection(); -} - -TEST_F( AwsIotConnectivityModuleTestAfterSuccessfulConnection, sendFileOverMQTT ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; - } - - int ret = std::system( "rm -rf ./Persistency && mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - - std::string testData = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; - const uint8_t *stringData = reinterpret_cast( testData.data() ); - - std::string filename = "testFile.bin"; - persistencyPtr->write( stringData, testData.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); - - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - std::list completeHandlers; - EXPECT_CALL( *mMqttClientWrapperMock, Publish( _, _ ) ) - .Times( 2 ) - .WillRepeatedly( - Invoke( [&completeHandlers]( - std::shared_ptr, - Aws::Crt::Mqtt5::OnPublishCompletionHandler onPublishCompletionCallback ) noexcept -> bool { - completeHandlers.push_back( std::move( onPublishCompletionCallback ) ); - return true; - } ) ); - - ASSERT_EQ( channel.sendFile( filename, testData.size() ), ConnectivityError::Success ); - - // Confirm 1st - completeHandlers.front().operator()( 0, std::make_shared() ); - completeHandlers.pop_front(); - - // Test callback return false - persistencyPtr->write( stringData, testData.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); - ASSERT_EQ( channel.sendFile( filename, testData.size() ), ConnectivityError::Success ); - - completeHandlers.front().operator()( 0, std::make_shared() ); - completeHandlers.pop_front(); - - channel.invalidateConnection(); -} - -/** @brief Test sending file over MQTT, no connection */ -TEST_F( AwsIotConnectivityModuleTest, sendFileOverMQTTNoConnection ) -{ - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) == nullptr ) - { - FAIL() << "Could not get the current working directory"; + ASSERT_EQ( memMgr.releaseReservedMemory( reservedMemory ), 0 ); } - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->init(); - const std::shared_ptr payloadManager = std::make_shared( persistencyPtr ); - AwsIotChannel channel( mConnectivityModule.get(), payloadManager, mMqttClientWrapper, "topic", false ); - std::string filename = "testFile.json"; - ASSERT_EQ( channel.sendFile( filename, 100 ), ConnectivityError::NoConnection ); - CollectionSchemeParams collectionSchemeParams; - collectionSchemeParams.persist = true; - collectionSchemeParams.compression = false; - ASSERT_EQ( channel.sendFile( filename, 100, collectionSchemeParams ), ConnectivityError::NoConnection ); - channel.invalidateConnection(); + sender.invalidateConnection(); } /** @brief Test the separate thread with exponential backoff that tries to connect until connection succeeds */ @@ -854,7 +939,7 @@ TEST_F( AwsIotConnectivityModuleTest, asyncConnect ) { EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithClientExtendedValidationAndFlowControl( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithConnectOptions( _ ) ).Times( 1 ); - EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithSessionBehavior( _ ) ).Times( 1 ); + EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithOfflineQueueBehavior( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, WithPingTimeoutMs( _ ) ).Times( 1 ); EXPECT_CALL( *mMqttClientBuilderWrapperMock, Build() ).Times( 1 ); // Don't automatically invoke the callback, we want to call it manually. @@ -865,7 +950,7 @@ TEST_F( AwsIotConnectivityModuleTest, asyncConnect ) std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); // first attempt should come immediately EXPECT_CALL( *mMqttClientWrapperMock, Start() ).WillOnce( Return( true ) ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).WillOnce( Return( true ) ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).WillOnce( Return( true ) ); { Aws::Crt::Mqtt5::OnConnectionFailureEventData eventData; eventData.errorCode = AWS_ERROR_MQTT_TIMEOUT; @@ -879,7 +964,7 @@ TEST_F( AwsIotConnectivityModuleTest, asyncConnect ) std::this_thread::sleep_for( std::chrono::milliseconds( 1100 ) ); // minimum wait time 1 second EXPECT_CALL( *mMqttClientWrapperMock, Start() ).WillOnce( Return( true ) ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).WillOnce( Return( true ) ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).WillOnce( Return( true ) ); { Aws::Crt::Mqtt5::OnConnectionFailureEventData eventData; eventData.errorCode = AWS_ERROR_MQTT_TIMEOUT; @@ -899,7 +984,7 @@ TEST_F( AwsIotConnectivityModuleTest, asyncConnect ) WAIT_ASSERT_TRUE( mConnectivityModule->isAlive() ); - EXPECT_CALL( *mMqttClientWrapperMock, Stop() ).Times( 1 ); + EXPECT_CALL( *mMqttClientWrapperMock, Stop( _ ) ).Times( 1 ); } } // namespace IoTFleetWise diff --git a/test/unit/CANDataSourceTest.cpp b/test/unit/CANDataSourceTest.cpp index 5b24e8cb..cb596318 100644 --- a/test/unit/CANDataSourceTest.cpp +++ b/test/unit/CANDataSourceTest.cpp @@ -7,6 +7,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "VehicleDataSourceTypes.h" #include "WaitUntil.h" @@ -141,7 +142,7 @@ class CANDataSourceTest : public ::testing::Test mSocketFD = setup(); if ( mSocketFD == -1 ) { - GTEST_SKIP() << "Skipping test fixture due to unavailability of socket"; + GTEST_FAIL() << "Test failed due to unavailability of socket"; } std::unordered_map frameMap; CANMessageDecoderMethod decoderMethod; @@ -158,6 +159,7 @@ class CANDataSourceTest : public ::testing::Test sigFormat1.mSizeInBits = 30; sigFormat1.mOffset = 0.0; sigFormat1.mFactor = 1.0; + sigFormat1.mSignalType = SignalType::DOUBLE; CANSignalFormat sigFormat2; sigFormat2.mSignalID = 7; @@ -167,6 +169,7 @@ class CANDataSourceTest : public ::testing::Test sigFormat2.mSizeInBits = 31; sigFormat2.mOffset = 0.0; sigFormat2.mFactor = 1.0; + sigFormat2.mSignalType = SignalType::DOUBLE; decoderMethod.format.mSignals.push_back( sigFormat1 ); decoderMethod.format.mSignals.push_back( sigFormat2 ); @@ -189,9 +192,10 @@ class CANDataSourceTest : public ::testing::Test TEST_F( CANDataSourceTest, invalidInit ) { - auto signalBufferPtr = std::make_shared( 10 ); - - CANDataConsumer consumer{ signalBufferPtr }; + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + CANDataConsumer consumer{ signalBufferDistributor }; CANDataSource dataSource{ INVALID_CAN_SOURCE_NUMERIC_ID, CanTimestampType::KERNEL_HARDWARE_TIMESTAMP, "vcan0", false, 100, consumer }; ASSERT_FALSE( dataSource.init() ); @@ -200,29 +204,33 @@ TEST_F( CANDataSourceTest, invalidInit ) TEST_F( CANDataSourceTest, testNoDecoderDictionary ) { ASSERT_TRUE( mSocketFD != -1 ); - auto signalBufferPtr = std::make_shared( 10 ); - - CANDataConsumer consumer{ signalBufferPtr }; + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + CANDataConsumer consumer{ signalBufferDistributor }; CANDataSource dataSource{ 0, CanTimestampType::KERNEL_HARDWARE_TIMESTAMP, "vcan0", false, 100, consumer }; ASSERT_TRUE( dataSource.init() ); ASSERT_TRUE( dataSource.isAlive() ); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); // No disconnect to test destructor disconnect } TEST_F( CANDataSourceTest, testValidDecoderDictionary ) { ASSERT_TRUE( mSocketFD != -1 ); - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); - CANDataConsumer consumer{ signalBufferPtr }; + CANDataConsumer consumer{ signalBufferDistributor }; CANDataSource dataSource{ 0, CanTimestampType::KERNEL_HARDWARE_TIMESTAMP, "vcan0", false, 100, consumer }; ASSERT_TRUE( dataSource.init() ); ASSERT_TRUE( dataSource.isAlive() ); dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( sendTestMessage( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( sendTestMessage( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); @@ -241,14 +249,14 @@ TEST_F( CANDataSourceTest, testValidDecoderDictionary ) } // Test message a different message ID is not received - DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD, 0x456 ) && signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD, 0x456 ) && signalBuffer->pop( collectedDataFrame ) ); // Test invalidation of decoder dictionary dataSource.onChangeOfActiveDictionary( nullptr, VehicleDataSourceProtocol::RAW_SOCKET ); - DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); // Check it ignores dictionaries for other protocols dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::OBD ); - DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( sendTestMessage( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); ASSERT_TRUE( dataSource.disconnect() ); } @@ -257,15 +265,18 @@ TEST_F( CANDataSourceTest, testCanFDSocketMode ) cleanUp( mSocketFD ); mSocketFD = setup( true ); ASSERT_TRUE( mSocketFD != -1 ); - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); - CANDataConsumer consumer{ signalBufferPtr }; + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + + CANDataConsumer consumer{ signalBufferDistributor }; CANDataSource dataSource{ 0, CanTimestampType::KERNEL_SOFTWARE_TIMESTAMP, "vcan0", false, 100, consumer }; ASSERT_TRUE( dataSource.init() ); ASSERT_TRUE( dataSource.isAlive() ); dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( sendTestFDMessage( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( sendTestFDMessage( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); @@ -288,15 +299,18 @@ TEST_F( CANDataSourceTest, testCanFDSocketMode ) TEST_F( CANDataSourceTest, testExtractExtendedID ) { ASSERT_TRUE( mSocketFD != -1 ); - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); - CANDataConsumer consumer{ signalBufferPtr }; + CANDataConsumer consumer{ signalBufferDistributor }; CANDataSource dataSource{ 0, CanTimestampType::KERNEL_HARDWARE_TIMESTAMP, "vcan0", false, 100, consumer }; ASSERT_TRUE( dataSource.init() ); ASSERT_TRUE( dataSource.isAlive() ); dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( sendTestMessageExtendedID( mSocketFD ) && signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( sendTestMessageExtendedID( mSocketFD ) && signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); diff --git a/test/unit/CANDecoderTest.cpp b/test/unit/CANDecoderTest.cpp index 0ead6456..3f969858 100644 --- a/test/unit/CANDecoderTest.cpp +++ b/test/unit/CANDecoderTest.cpp @@ -58,8 +58,8 @@ TEST( CANDecoderTest, CANDecoderTestSimpleMessage ) std::unordered_set signalIDsToCollect = { 1, 7 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 2 ); - ASSERT_EQ( decodedSignals[0].mRawValue, 13 ); - ASSERT_EQ( decodedSignals[1].mRawValue, 4084 ); + ASSERT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 13 ); + ASSERT_DOUBLE_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 408.4 ); } TEST( CANDecoderTest, CANDecoderTestSimpleMessage2 ) @@ -104,8 +104,8 @@ TEST( CANDecoderTest, CANDecoderTestSimpleMessage2 ) std::unordered_set signalIDsToCollect = { 1, 7 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 2 ); - ASSERT_EQ( decodedSignals[0].mRawValue, 153638137 ); - ASSERT_EQ( decodedSignals[1].mRawValue, -299667802 ); + ASSERT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 153638137 ); + ASSERT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, -299667802 ); } // Precision Test @@ -252,9 +252,9 @@ TEST( CANDecoderTest, CANDecoderTestSimpleMessage3 ) std::unordered_set signalIDsToCollect = { 1, 2, 3 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 3 ); - EXPECT_EQ( decodedSignals[0].mRawValue, 0x2301 ); - EXPECT_EQ( decodedSignals[1].mRawValue, 0x4567 ); - EXPECT_EQ( decodedSignals[2].mRawValue, 0x89AB ); + EXPECT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 0x2301 ); + EXPECT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 0x4567 ); + EXPECT_EQ( decodedSignals[2].mPhysicalValue.signalValue.doubleVal, 0x89AB ); } TEST( CANDecoderTest, CANDecoderTestSimpleCanFdMessage ) { @@ -317,10 +317,10 @@ TEST( CANDecoderTest, CANDecoderTestSimpleCanFdMessage ) std::unordered_set signalIDsToCollect = { 1, 2, 3, 4 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 64, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 4 ); - EXPECT_EQ( decodedSignals[0].mRawValue, 0x2301 ); - EXPECT_EQ( decodedSignals[1].mRawValue, 0x70123456 ); - EXPECT_EQ( decodedSignals[2].mRawValue, 0x456701 ); - EXPECT_EQ( decodedSignals[3].mRawValue, 0x7012 ); + EXPECT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 0x2301 ); + EXPECT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 0x70123456 ); + EXPECT_EQ( decodedSignals[2].mPhysicalValue.signalValue.doubleVal, 0x456701 ); + EXPECT_EQ( decodedSignals[3].mPhysicalValue.signalValue.doubleVal, 0x7012 ); } TEST( CANDecoderTest, CANDecoderTestOnlyDecodeSomeSignals ) @@ -374,8 +374,8 @@ TEST( CANDecoderTest, CANDecoderTestOnlyDecodeSomeSignals ) std::unordered_set signalIDsToCollect = { 1, 3 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 2 ); - EXPECT_EQ( decodedSignals[0].mRawValue, 0x2301 ); - EXPECT_EQ( decodedSignals[1].mRawValue, 0x89AB ); + EXPECT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 0x2301 ); + EXPECT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 0x89AB ); } TEST( CANDecoderTest, CANDecoderTestMultiplexedMessage1 ) @@ -450,10 +450,10 @@ TEST( CANDecoderTest, CANDecoderTestMultiplexedMessage1 ) std::unordered_set signalIDsToCollect = { 50, 51, 52, 53 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 4 ); - EXPECT_EQ( decodedSignals[0].mRawValue, 0x05 ); - EXPECT_EQ( decodedSignals[1].mRawValue, 0x4B ); - EXPECT_EQ( decodedSignals[2].mRawValue, 0x20B ); - EXPECT_EQ( decodedSignals[3].mRawValue, 0xD3 ); + EXPECT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 0x05 ); + EXPECT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 0x4B ); + EXPECT_EQ( decodedSignals[2].mPhysicalValue.signalValue.doubleVal, 0x20B ); + EXPECT_EQ( decodedSignals[3].mPhysicalValue.signalValue.doubleVal, 0xD3 ); } TEST( CANDecoderTest, CANDecoderTestMultiplexedMessage2 ) @@ -542,10 +542,10 @@ TEST( CANDecoderTest, CANDecoderTestMultiplexedMessage2 ) std::unordered_set signalIDsToCollect = { 54, 55, 56, 57, 58 }; ASSERT_TRUE( decoder.decodeCANMessage( frameData.data(), 8, msgFormat, signalIDsToCollect, decodedSignals ) ); ASSERT_EQ( decodedSignals.size(), 4 ); - EXPECT_EQ( decodedSignals[0].mRawValue, 0x06 ); - EXPECT_EQ( decodedSignals[1].mRawValue, 0x1B ); - EXPECT_EQ( decodedSignals[2].mRawValue, 0x3EB ); - EXPECT_EQ( decodedSignals[3].mRawValue, 0xE3B ); + EXPECT_EQ( decodedSignals[0].mPhysicalValue.signalValue.doubleVal, 0x06 ); + EXPECT_EQ( decodedSignals[1].mPhysicalValue.signalValue.doubleVal, 0x1B ); + EXPECT_EQ( decodedSignals[2].mPhysicalValue.signalValue.doubleVal, 0x3EB ); + EXPECT_EQ( decodedSignals[3].mPhysicalValue.signalValue.doubleVal, 0xE3B ); } TEST( CANDecoderTest, CANDecoderTestInvalidSignalLayout ) diff --git a/test/unit/CacheAndPersistTest.cpp b/test/unit/CacheAndPersistTest.cpp index a072dbf6..a638d140 100644 --- a/test/unit/CacheAndPersistTest.cpp +++ b/test/unit/CacheAndPersistTest.cpp @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 #include "CacheAndPersist.h" -#include +#include "Testing.h" +#include #include #include #include #include #include #include -#include #include namespace Aws @@ -20,486 +20,309 @@ namespace IoTFleetWise // Unit Tests for the collectionScheme persistency TEST( CacheAndPersistTest, testCollectionSchemePersistency ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - ASSERT_TRUE( storage.init() ); - // Delete any existing contents - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - - // create a test obj - std::string testString = "Test CollectionScheme"; - size_t size = testString.size(); - - ASSERT_EQ( storage.write( reinterpret_cast( testString.c_str() ), - size, - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), size ); - - std::unique_ptr readBufPtr( new uint8_t[size]() ); - ASSERT_EQ( storage.read( readBufPtr.get(), size, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - - std::string out( reinterpret_cast( readBufPtr.get() ), size ); - - std::cout << "File contents: " << out << std::endl; - ASSERT_STREQ( out.c_str(), testString.c_str() ); - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), 0 ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + // create a test obj + std::string testString = "Test CollectionScheme"; + size_t size = testString.size(); + + ASSERT_EQ( storage->write( + reinterpret_cast( testString.c_str() ), size, DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), size ); + + std::unique_ptr readBufPtr( new uint8_t[size]() ); + ASSERT_EQ( storage->read( readBufPtr.get(), size, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + + std::string out( reinterpret_cast( readBufPtr.get() ), size ); + + std::cout << "File contents: " << out << std::endl; + ASSERT_STREQ( out.c_str(), testString.c_str() ); + ASSERT_EQ( storage->erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), 0 ); } // Unit Tests for the DM persistency TEST( CacheAndPersistTest, testDecoderManifestPersistency ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - ASSERT_TRUE( storage.init() ); - // create a test obj - std::string testString = "Test Decoder Manifest"; - size_t size = testString.size(); - - ASSERT_EQ( - storage.write( reinterpret_cast( testString.c_str() ), size, DataType::DECODER_MANIFEST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::DECODER_MANIFEST ), size ); - - std::unique_ptr readBufPtr( new uint8_t[size]() ); - ASSERT_EQ( storage.read( readBufPtr.get(), size, DataType::DECODER_MANIFEST ), ErrorCode::SUCCESS ); - std::string out( reinterpret_cast( readBufPtr.get() ), size ); - std::cout << "File contents: " << out << std::endl; - ASSERT_STREQ( out.c_str(), testString.c_str() ); - ASSERT_EQ( storage.erase( DataType::DECODER_MANIFEST ), ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::DECODER_MANIFEST ), 0 ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + // create a test obj + std::string testString = "Test Decoder Manifest"; + size_t size = testString.size(); + + ASSERT_EQ( + storage->write( reinterpret_cast( testString.c_str() ), size, DataType::DECODER_MANIFEST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::DECODER_MANIFEST ), size ); + + std::unique_ptr readBufPtr( new uint8_t[size]() ); + ASSERT_EQ( storage->read( readBufPtr.get(), size, DataType::DECODER_MANIFEST ), ErrorCode::SUCCESS ); + std::string out( reinterpret_cast( readBufPtr.get() ), size ); + std::cout << "File contents: " << out << std::endl; + ASSERT_STREQ( out.c_str(), testString.c_str() ); + ASSERT_EQ( storage->erase( DataType::DECODER_MANIFEST ), ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::DECODER_MANIFEST ), 0 ); } TEST( CacheAndPersistTest, testWriteEmptyBuffer ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); - ASSERT_EQ( storage.write( nullptr, 0, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); + auto storage = createCacheAndPersist(); - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + ASSERT_TRUE( storage->init() ); + ASSERT_EQ( storage->write( nullptr, 0, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); } // Check for the memory full condition TEST( CacheAndPersistTest, testMemoryFull ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 2 ); - - ASSERT_TRUE( storage.init() ); - // Delete any existing contents - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - - // create a test obj - std::string testString = "test123"; - - ASSERT_EQ( storage.write( reinterpret_cast( testString.c_str() ), - testString.size(), - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::MEMORY_FULL ); - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), 0 ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = std::make_shared( getTempDir().string(), 2 ); + + // Delete any existing contents + ASSERT_EQ( storage->erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + + // create a test obj + std::string testString = "test123"; + + ASSERT_EQ( storage->write( reinterpret_cast( testString.c_str() ), + testString.size(), + DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::MEMORY_FULL ); + ASSERT_EQ( storage->erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), 0 ); } // Check for the invalid data type condition TEST( CacheAndPersistTest, testInvalidDataType ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); + auto storage = createCacheAndPersist(); - std::string testString = "test invalid data type"; - size_t size = testString.size(); - ASSERT_EQ( - storage.write( reinterpret_cast( testString.c_str() ), size, DataType::DEFAULT_DATA_TYPE ), - ErrorCode::INVALID_DATATYPE ); + std::string testString = "test invalid data type"; + size_t size = testString.size(); + ASSERT_EQ( + storage->write( reinterpret_cast( testString.c_str() ), size, DataType::DEFAULT_DATA_TYPE ), + ErrorCode::INVALID_DATATYPE ); - // Reading back the data - std::unique_ptr readBufPtr( new uint8_t[size]() ); + // Reading back the data + std::unique_ptr readBufPtr( new uint8_t[size]() ); - ASSERT_EQ( storage.getSize( DataType::DEFAULT_DATA_TYPE ), INVALID_FILE_SIZE ); - ASSERT_EQ( storage.read( readBufPtr.get(), size, DataType::DEFAULT_DATA_TYPE ), ErrorCode::INVALID_DATATYPE ); - ASSERT_EQ( storage.erase( DataType::DEFAULT_DATA_TYPE ), ErrorCode::INVALID_DATATYPE ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + ASSERT_EQ( storage->getSize( DataType::DEFAULT_DATA_TYPE ), INVALID_FILE_SIZE ); + ASSERT_EQ( storage->read( readBufPtr.get(), size, DataType::DEFAULT_DATA_TYPE ), ErrorCode::INVALID_DATATYPE ); + ASSERT_EQ( storage->erase( DataType::DEFAULT_DATA_TYPE ), ErrorCode::INVALID_DATATYPE ); } // Tests writing multiple collectionSchemes, checks if it gets overwritten TEST( CacheAndPersistTest, testCollectionSchemeListOverwrite ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); - - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - - // create a test obj - std::string testString1 = "CollectionScheme 1234"; - std::string testString2 = "CollectionScheme 5"; - std::string testString3 = "CollectionScheme 678"; - - size_t size1 = testString1.size(); - ASSERT_EQ( storage.write( reinterpret_cast( testString1.c_str() ), - size1, - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), size1 ); - - std::unique_ptr readBufPtr1( new uint8_t[size1]() ); - ASSERT_EQ( storage.read( readBufPtr1.get(), size1, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - std::string out1( reinterpret_cast( readBufPtr1.get() ), size1 ); - ASSERT_STREQ( out1.c_str(), testString1.c_str() ); - - size_t size2 = testString2.size(); - - ASSERT_EQ( storage.write( reinterpret_cast( testString2.c_str() ), - size2, - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), size2 ); - - std::unique_ptr readBufPtr2( new uint8_t[size2]() ); - ASSERT_EQ( storage.read( readBufPtr2.get(), size2, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - std::string out2( reinterpret_cast( readBufPtr2.get() ), size2 ); - ASSERT_STREQ( out2.c_str(), testString2.c_str() ); - - size_t size3 = testString3.size(); - - ASSERT_EQ( storage.write( reinterpret_cast( testString3.c_str() ), - size3, - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), size3 ); - - std::unique_ptr readBufPtr3( new uint8_t[size3]() ); - ASSERT_EQ( storage.read( readBufPtr3.get(), size3, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - std::string out3( reinterpret_cast( readBufPtr3.get() ), size3 ); - ASSERT_STREQ( out3.c_str(), testString3.c_str() ); - - ASSERT_EQ( storage.erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + + // create a test obj + std::string testString1 = "CollectionScheme 1234"; + std::string testString2 = "CollectionScheme 5"; + std::string testString3 = "CollectionScheme 678"; + + size_t size1 = testString1.size(); + ASSERT_EQ( storage->write( + reinterpret_cast( testString1.c_str() ), size1, DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), size1 ); + + std::unique_ptr readBufPtr1( new uint8_t[size1]() ); + ASSERT_EQ( storage->read( readBufPtr1.get(), size1, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + std::string out1( reinterpret_cast( readBufPtr1.get() ), size1 ); + ASSERT_STREQ( out1.c_str(), testString1.c_str() ); + + size_t size2 = testString2.size(); + + ASSERT_EQ( storage->write( + reinterpret_cast( testString2.c_str() ), size2, DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), size2 ); + + std::unique_ptr readBufPtr2( new uint8_t[size2]() ); + ASSERT_EQ( storage->read( readBufPtr2.get(), size2, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + std::string out2( reinterpret_cast( readBufPtr2.get() ), size2 ); + ASSERT_STREQ( out2.c_str(), testString2.c_str() ); + + size_t size3 = testString3.size(); + + ASSERT_EQ( storage->write( + reinterpret_cast( testString3.c_str() ), size3, DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), size3 ); + + std::unique_ptr readBufPtr3( new uint8_t[size3]() ); + ASSERT_EQ( storage->read( readBufPtr3.get(), size3, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); + std::string out3( reinterpret_cast( readBufPtr3.get() ), size3 ); + ASSERT_STREQ( out3.c_str(), testString3.c_str() ); + + ASSERT_EQ( storage->erase( DataType::COLLECTION_SCHEME_LIST ), ErrorCode::SUCCESS ); } // Tests reading an empty file TEST( CacheAndPersistTest, testReadEmptyFile ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); + auto storage = createCacheAndPersist(); - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); - - std::unique_ptr readBufPtr( new uint8_t[100]() ); - ASSERT_EQ( storage.read( readBufPtr.get(), 100, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::EMPTY ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + std::unique_ptr readBufPtr( new uint8_t[100]() ); + ASSERT_EQ( storage->read( readBufPtr.get(), 100, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::EMPTY ); } // Tests reading a big file TEST( CacheAndPersistTest, testReadBigFile ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); + auto storage = createCacheAndPersist(); - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); + ASSERT_TRUE( storage->init() ); + size_t size = 200000; + std::unique_ptr readBufPtr( new uint8_t[size]() ); - ASSERT_TRUE( storage.init() ); - size_t size = 200000; - std::unique_ptr readBufPtr( new uint8_t[size]() ); - - ASSERT_EQ( storage.read( readBufPtr.get(), size, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::MEMORY_FULL ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + ASSERT_EQ( storage->read( readBufPtr.get(), size, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::MEMORY_FULL ); } // Tests read with size mismatch TEST( CacheAndPersistTest, testReadSizeMismatch ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); + auto storage = createCacheAndPersist(); - ASSERT_TRUE( storage.init() ); + std::string testString = "Wrong size test case"; + size_t size = testString.size(); - std::string testString = "Wrong size test case"; - size_t size = testString.size(); + ASSERT_EQ( storage->write( + reinterpret_cast( testString.c_str() ), size, DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::COLLECTION_SCHEME_LIST ), size ); - ASSERT_EQ( storage.write( reinterpret_cast( testString.c_str() ), - size, - DataType::COLLECTION_SCHEME_LIST ), - ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::COLLECTION_SCHEME_LIST ), size ); + std::unique_ptr readBufPtr( new uint8_t[size]() ); - std::unique_ptr readBufPtr( new uint8_t[size]() ); - - ASSERT_EQ( storage.read( readBufPtr.get(), 123, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + ASSERT_EQ( storage->read( readBufPtr.get(), 123, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); } // Test Data Persistency TEST( CacheAndPersistTest, testDataPersistency ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - // Delete any existing contents - ASSERT_TRUE( storage.init() ); - - // create a test obj - std::string testString = "Store this data - 1"; - size_t size = testString.size(); - std::cout << "String size: " << size << std::endl; - - std::string filename = "testfile.bin"; - ASSERT_EQ( storage.write( reinterpret_cast( testString.c_str() ), - size, - DataType::EDGE_TO_CLOUD_PAYLOAD, - filename ), - ErrorCode::SUCCESS ); - - size_t readSize = storage.getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); - std::cout << "testDataPersistency::readSize: " << readSize << std::endl; - - std::unique_ptr readBufPtr( new uint8_t[readSize]() ); - ASSERT_EQ( storage.read( readBufPtr.get(), readSize, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ), - ErrorCode::SUCCESS ); - - std::string out( reinterpret_cast( readBufPtr.get() ), readSize ); - ASSERT_STREQ( out.c_str(), testString.c_str() ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + + // create a test obj + std::string testString = "Store this data - 1"; + size_t size = testString.size(); + std::cout << "String size: " << size << std::endl; + + std::string filename = "testfile.bin"; + ASSERT_EQ( + storage->write( + reinterpret_cast( testString.c_str() ), size, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ), + ErrorCode::SUCCESS ); + + size_t readSize = storage->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename ); + std::cout << "testDataPersistency::readSize: " << readSize << std::endl; + + std::unique_ptr readBufPtr( new uint8_t[readSize]() ); + ASSERT_EQ( storage->read( readBufPtr.get(), readSize, DataType::EDGE_TO_CLOUD_PAYLOAD, filename ), + ErrorCode::SUCCESS ); + + std::string out( reinterpret_cast( readBufPtr.get() ), readSize ); + ASSERT_STREQ( out.c_str(), testString.c_str() ); } // Test multiple events being logged while the car is offline TEST( CacheAndPersistTest, testMultipleEventDataPersistency ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_EQ( storage.erase( DataType::PAYLOAD_METADATA ), ErrorCode::SUCCESS ); - ASSERT_TRUE( storage.init() ); - - // create a test obj - std::string testString1 = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; - size_t size1 = testString1.size(); - std::cout << "String size: " << size1 << std::endl; - - std::string filename1 = "testfile1.bin"; - ASSERT_EQ( storage.write( reinterpret_cast( testString1.c_str() ), - size1, - DataType::EDGE_TO_CLOUD_PAYLOAD, - filename1 ), - ErrorCode::SUCCESS ); - - std::string testString2 = "34567089!@#$%^&*()_+?><"; - size_t size2 = testString2.size(); - std::cout << "String size: " << size2 << std::endl; - - std::string filename2 = "testfile2.bin"; - ASSERT_EQ( storage.write( reinterpret_cast( testString2.c_str() ), - size2, - DataType::EDGE_TO_CLOUD_PAYLOAD, - filename2 ), - ErrorCode::SUCCESS ); - - std::vector payload1( size1 ); - ASSERT_EQ( storage.read( payload1.data(), payload1.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), - ErrorCode::SUCCESS ); - std::string out1( payload1.begin(), payload1.end() ); - ASSERT_STREQ( out1.c_str(), testString1.c_str() ); - - ASSERT_EQ( storage.erase( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), 0 ); - - std::vector payload2( size2 ); - ASSERT_EQ( storage.read( payload2.data(), payload2.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), - ErrorCode::SUCCESS ); - std::string out2( payload2.begin(), payload2.end() ); - ASSERT_STREQ( out2.c_str(), testString2.c_str() ); - - ASSERT_EQ( storage.erase( DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), ErrorCode::SUCCESS ); - ASSERT_EQ( storage.getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), 0 ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + + auto storage = createCacheAndPersist(); + + // create a test obj + std::string testString1 = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; + size_t size1 = testString1.size(); + std::cout << "String size: " << size1 << std::endl; + + std::string filename1 = "testfile1.bin"; + ASSERT_EQ( storage->write( reinterpret_cast( testString1.c_str() ), + size1, + DataType::EDGE_TO_CLOUD_PAYLOAD, + filename1 ), + ErrorCode::SUCCESS ); + + std::string testString2 = "34567089!@#$%^&*()_+?><"; + size_t size2 = testString2.size(); + std::cout << "String size: " << size2 << std::endl; + + std::string filename2 = "testfile2.bin"; + ASSERT_EQ( storage->write( reinterpret_cast( testString2.c_str() ), + size2, + DataType::EDGE_TO_CLOUD_PAYLOAD, + filename2 ), + ErrorCode::SUCCESS ); + + std::vector payload1( size1 ); + ASSERT_EQ( storage->read( payload1.data(), payload1.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), + ErrorCode::SUCCESS ); + std::string out1( payload1.begin(), payload1.end() ); + ASSERT_STREQ( out1.c_str(), testString1.c_str() ); + + ASSERT_EQ( storage->erase( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), 0 ); + + std::vector payload2( size2 ); + ASSERT_EQ( storage->read( payload2.data(), payload2.size(), DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), + ErrorCode::SUCCESS ); + std::string out2( payload2.begin(), payload2.end() ); + ASSERT_STREQ( out2.c_str(), testString2.c_str() ); + + ASSERT_EQ( storage->erase( DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), ErrorCode::SUCCESS ); + ASSERT_EQ( storage->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename2 ), 0 ); } TEST( CacheAndPersistTest, testReadEmptyBuffer ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); - ASSERT_EQ( storage.read( nullptr, 0, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + ASSERT_EQ( storage->read( nullptr, 0, DataType::COLLECTION_SCHEME_LIST ), ErrorCode::INVALID_DATA ); } TEST( CacheAndPersistTest, testNoFilenameForPayload ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); + auto storage = createCacheAndPersist(); - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); + std::string testString = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; + size_t size = testString.size(); - ASSERT_TRUE( storage.init() ); + std::vector payload( size ); - std::string testString = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; - size_t size = testString.size(); - - std::vector payload( size ); - - ASSERT_EQ( storage.write( - reinterpret_cast( testString.c_str() ), size, DataType::EDGE_TO_CLOUD_PAYLOAD ), - ErrorCode::INVALID_DATATYPE ); - ASSERT_EQ( storage.getSize( DataType::EDGE_TO_CLOUD_PAYLOAD ), INVALID_FILE_SIZE ); - ASSERT_EQ( storage.read( payload.data(), payload.size(), DataType::EDGE_TO_CLOUD_PAYLOAD ), - ErrorCode::INVALID_DATATYPE ); - ASSERT_EQ( storage.erase( DataType::EDGE_TO_CLOUD_PAYLOAD ), ErrorCode::INVALID_DATATYPE ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + ASSERT_EQ( storage->write( + reinterpret_cast( testString.c_str() ), size, DataType::EDGE_TO_CLOUD_PAYLOAD ), + ErrorCode::INVALID_DATATYPE ); + ASSERT_EQ( storage->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD ), INVALID_FILE_SIZE ); + ASSERT_EQ( storage->read( payload.data(), payload.size(), DataType::EDGE_TO_CLOUD_PAYLOAD ), + ErrorCode::INVALID_DATATYPE ); + ASSERT_EQ( storage->erase( DataType::EDGE_TO_CLOUD_PAYLOAD ), ErrorCode::INVALID_DATATYPE ); } TEST( CacheAndPersistTest, testCleanUpFunction ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - - ASSERT_TRUE( storage.init() ); - - // create a test obj - std::string testString1 = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; - size_t size1 = testString1.size(); - std::cout << "String size: " << size1 << std::endl; - - std::string filename1 = "testfile1.bin"; - ASSERT_EQ( storage.write( reinterpret_cast( testString1.c_str() ), - size1, - DataType::EDGE_TO_CLOUD_PAYLOAD, - filename1 ), - ErrorCode::SUCCESS ); - storage.clearMetadata(); - } + auto storage = createCacheAndPersist(); + + // create a test obj + std::string testString1 = "abcdefjh!24$iklmnop!24$3@qrstuvwxyz"; + size_t size1 = testString1.size(); + std::cout << "String size: " << size1 << std::endl; + + std::string filename1 = "testfile1.bin"; + ASSERT_EQ( storage->write( reinterpret_cast( testString1.c_str() ), + size1, + DataType::EDGE_TO_CLOUD_PAYLOAD, + filename1 ), + ErrorCode::SUCCESS ); + storage->clearMetadata(); } TEST( CacheAndPersistTest, testInitAfterCleanUpFunction ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - CacheAndPersist storage( std::string( buffer ) + "/Persistency", 131072 ); - ASSERT_TRUE( storage.init() ); - - std::string filename1 = "testfile1.bin"; - - ASSERT_EQ( storage.getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), 0 ); - ASSERT_EQ( - storage.getSize( DataType::PAYLOAD_METADATA ), - 30 ); // 30 is a size for a default empty JSON metadata scheme, containing the version and the empty array - - int ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto storage = createCacheAndPersist(); + + std::string filename1 = "testfile1.bin"; + + ASSERT_EQ( storage->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename1 ), 0 ); + ASSERT_EQ( + storage->getSize( DataType::PAYLOAD_METADATA ), + 30 ); // 30 is a size for a default empty JSON metadata scheme, containing the version and the empty array } } // namespace IoTFleetWise diff --git a/test/unit/CollectionInspectionEngineTest.cpp b/test/unit/CollectionInspectionEngineTest.cpp index dadea430..0d26e37e 100644 --- a/test/unit/CollectionInspectionEngineTest.cpp +++ b/test/unit/CollectionInspectionEngineTest.cpp @@ -5,6 +5,7 @@ #include "CANDataTypes.h" #include "CollectionInspectionAPITypes.h" #include "ICollectionScheme.h" +#include "LogLevel.h" #include "OBDDataTypes.h" #include "SignalTypes.h" #include "Testing.h" @@ -70,8 +71,10 @@ class CollectionInspectionEngineTest : public ::testing::Test return static_cast( sigVal ) == signalValueWrapper.value.doubleVal; case SignalType::BOOLEAN: return static_cast( sigVal ) == signalValueWrapper.value.boolVal; + case SignalType::UNKNOWN: + return false; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA - case SignalType::RAW_DATA_BUFFER_HANDLE: + case SignalType::COMPLEX_SIGNAL: return static_cast( sigVal ) == signalValueWrapper.value.uint32Val; #endif } @@ -217,19 +220,19 @@ class CollectionInspectionEngineTest : public ::testing::Test multiply1->left = lastMax1.get(); multiply1->right = previousLastMax1.get(); - lastMin1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + lastMin1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; lastMin1->function.windowFunction = WindowFunction::LAST_FIXED_WINDOW_MIN; lastMin1->signalID = id1; - previousLastMin1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + previousLastMin1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; previousLastMin1->function.windowFunction = WindowFunction::PREV_LAST_FIXED_WINDOW_MIN; previousLastMin1->signalID = id1; - lastMax1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + lastMax1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; lastMax1->function.windowFunction = WindowFunction::LAST_FIXED_WINDOW_MAX; lastMax1->signalID = id1; - previousLastMax1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + previousLastMax1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; previousLastMax1->function.windowFunction = WindowFunction::PREV_LAST_FIXED_WINDOW_MAX; previousLastMax1->signalID = id1; @@ -338,7 +341,7 @@ class CollectionInspectionEngineTest : public ::testing::Test bigger1->left = function1.get(); bigger1->right = value1.get(); - function1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + function1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; function1->signalID = id1; function1->function.windowFunction = WindowFunction::LAST_FIXED_WINDOW_AVG; @@ -362,7 +365,7 @@ class CollectionInspectionEngineTest : public ::testing::Test bigger1->left = function1.get(); bigger1->right = value1.get(); - function1->nodeType = ExpressionNodeType::WINDOWFUNCTION; + function1->nodeType = ExpressionNodeType::WINDOW_FUNCTION; function1->signalID = id1; function1->function.windowFunction = WindowFunction::PREV_LAST_FIXED_WINDOW_AVG; @@ -382,7 +385,9 @@ class CollectionInspectionEngineTest : public ::testing::Test collectionSchemes->conditions.resize( 2 ); collectionSchemes->conditions[0].condition = getAlwaysFalseCondition().get(); + collectionSchemes->conditions[0].isStaticCondition = true; collectionSchemes->conditions[1].condition = getAlwaysFalseCondition().get(); + collectionSchemes->conditions[1].isStaticCondition = true; } void @@ -406,12 +411,14 @@ TYPED_TEST( CollectionInspectionEngineTest, TwoSignalsInConditionAndOneSignalToC s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; s1.isConditionOnlySignal = true; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 2; s2.sampleBufferSize = 50; s2.minimumSampleIntervalMs = 10; s2.fixedWindowPeriod = 77777; s2.isConditionOnlySignal = true; + s2.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s3{}; s3.signalID = 3; s3.sampleBufferSize = 50; @@ -425,6 +432,8 @@ TYPED_TEST( CollectionInspectionEngineTest, TwoSignalsInConditionAndOneSignalToC this->addSignalToCollect( this->collectionSchemes->conditions[0], s3 ); // The condition is (signalID(1)>-100) && (signalID(2)>-500) + // Condition contains signals + this->collectionSchemes->conditions[0].isStaticCondition = false; this->collectionSchemes->conditions[0].condition = this->getTwoSignalsBiggerCondition( s1.signalID, -100.0, s2.signalID, -500.0 ).get(); @@ -434,30 +443,30 @@ TYPED_TEST( CollectionInspectionEngineTest, TwoSignalsInConditionAndOneSignalToC TypeParam testVal1 = 10; TypeParam testVal2 = 20; TypeParam testVal3 = 30; - engine.addNewSignal( s3.signalID, timestamp, testVal1 ); - engine.addNewSignal( s3.signalID, timestamp, testVal2 ); - engine.addNewSignal( s3.signalID, timestamp, testVal3 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, testVal1 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, testVal2 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, testVal3 ); // Signals for condition are not available yet so collectionScheme should not trigger - engine.evaluateConditions( timestamp ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); timestamp += 1000; - engine.addNewSignal( s1.signalID, timestamp, -90.0 ); - engine.addNewSignal( s2.signalID, timestamp, -1000.0 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -90.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -1000.0 ); // Condition only for first signal is fulfilled (-90 > -100) but second not so boolean and is false - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); timestamp += 1000; - engine.addNewSignal( s2.signalID, timestamp, -480.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -480.0 ); // Condition is fulfilled so it should trigger - engine.evaluateConditions( timestamp ); - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 3 ); ASSERT_EQ( collectedData->signals[0].signalID, s3.signalID ); @@ -475,6 +484,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, EndlessCondition ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); ExpressionNode endless; @@ -485,13 +495,14 @@ TEST_F( CollectionInspectionEngineDoubleTest, EndlessCondition ) TimePoint timestamp = { 160000000, 100 }; + // Condition is static by default collectionSchemes->conditions[0].condition = &endless; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.evaluateConditions( timestamp ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - EXPECT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + EXPECT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, TooBigForSignalBuffer ) @@ -506,19 +517,21 @@ TEST_F( CollectionInspectionEngineDoubleTest, TooBigForSignalBuffer ) s1.fixedWindowPeriod = 77777; s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static be default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); // All signal come at the same timestamp - engine.addNewSignal( s1.signalID, timestamp + 1000, 0.2 ); - engine.addNewSignal( s1.signalID, timestamp + 2000, 0.3 ); + engine.addNewSignal( + s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); // All signal come at the same timestamp + engine.addNewSignal( s1.signalID, timestamp + 1000, timestamp.monotonicTimeMs + 1000, 0.2 ); + engine.addNewSignal( s1.signalID, timestamp + 2000, timestamp.monotonicTimeMs + 2000, 0.3 ); - engine.evaluateConditions( timestamp + 3000 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 3000 ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp + 5000, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp + 5000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 0 ); } @@ -532,6 +545,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, TooBigForSignalBufferOverflow ) s1.sampleBufferSize = 536870912; s1.minimumSampleIntervalMs = 5; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); InspectionMatrixCanFrameCollectionInfo c1; c1.frameID = 0x380; @@ -541,20 +555,21 @@ TEST_F( CollectionInspectionEngineDoubleTest, TooBigForSignalBufferOverflow ) collectionSchemes->conditions[0].canFrames.push_back( c1 ); collectionSchemes->conditions[0].minimumPublishIntervalMs = 500; + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // All signal come at the same timestamp - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.addNewSignal( s1.signalID, timestamp + 1000, 0.2 ); - engine.addNewSignal( s1.signalID, timestamp + 2000, 0.3 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp + 1000, timestamp.monotonicTimeMs + 1000, 0.2 ); + engine.addNewSignal( s1.signalID, timestamp + 2000, timestamp.monotonicTimeMs + 2000, 0.3 ); - engine.evaluateConditions( timestamp + 3000 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 3000 ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp + 4000, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp + 4000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 0 ); } @@ -568,27 +583,29 @@ TEST_F( CollectionInspectionEngineDoubleTest, SignalBufferErasedAfterNewConditio s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // All signal come at the same timestamp - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.addNewSignal( s1.signalID, timestamp + 1, 0.2 ); - engine.addNewSignal( s1.signalID, timestamp + 2, 0.3 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp + 1, timestamp.monotonicTimeMs + 1, 0.2 ); + engine.addNewSignal( s1.signalID, timestamp + 2, timestamp.monotonicTimeMs + 2, 0.3 ); // This call will flush the signal history buffer even when // exactly the same conditions are handed over. engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp + 3 ); - engine.addNewSignal( s1.signalID, timestamp + 3, 0.4 ); + engine.addNewSignal( s1.signalID, timestamp + 3, timestamp.monotonicTimeMs + 3, 0.4 ); - engine.evaluateConditions( timestamp + 3 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 3 ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp + 3, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp + 3, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 1 ); @@ -604,21 +621,23 @@ TEST_F( CollectionInspectionEngineDoubleTest, CollectBurstWithoutSubsampling ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // All signal come at the same timestamp - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.addNewSignal( s1.signalID, timestamp, 0.2 ); - engine.addNewSignal( s1.signalID, timestamp, 0.3 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.2 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.3 ); - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 3 ); @@ -635,8 +654,10 @@ TEST_F( CollectionInspectionEngineDoubleTest, IllegalSignalID ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -651,8 +672,9 @@ TEST_F( CollectionInspectionEngineDoubleTest, IllegalSampleSize ) s1.sampleBufferSize = 0; // Not allowed s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); - + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -663,6 +685,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, ZeroSignalsOnlyDTCCollection ) { CollectionInspectionEngine engine; collectionSchemes->conditions[0].includeActiveDtcs = true; + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -674,9 +697,9 @@ TEST_F( CollectionInspectionEngineDoubleTest, ZeroSignalsOnlyDTCCollection ) dtcInfo.receiveTime = timestamp.systemTimeMs; engine.setActiveDTCs( dtcInfo ); timestamp += 1000; - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); EXPECT_EQ( collectedData->mDTCInfo.mDTCCodes[0], "B1217" ); } @@ -691,7 +714,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, CollectRawCanFrames ) c1.sampleBufferSize = 10; c1.minimumSampleIntervalMs = 0; collectionSchemes->conditions[0].canFrames.push_back( c1 ); - + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -700,10 +723,10 @@ TEST_F( CollectionInspectionEngineDoubleTest, CollectRawCanFrames ) std::array buf = { 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; engine.addNewRawCanFrame( c1.frameID, c1.channelID, timestamp, buf, sizeof( buf ) ); - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->canFrames.size(), 1 ); @@ -723,7 +746,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, CollectRawCanFDFrames ) c1.sampleBufferSize = 10; c1.minimumSampleIntervalMs = 0; collectionSchemes->conditions[0].canFrames.push_back( c1 ); - + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -733,10 +756,10 @@ TEST_F( CollectionInspectionEngineDoubleTest, CollectRawCanFDFrames ) 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0, 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; engine.addNewRawCanFrame( c1.frameID, c1.channelID, timestamp, buf, sizeof( buf ) ); - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->canFrames.size(), 1 ); @@ -768,6 +791,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleCanSubsampling ) collectionSchemes->conditions[1].canFrames.push_back( c1 ); collectionSchemes->conditions[1].canFrames.push_back( c2 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -790,10 +814,10 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleCanSubsampling ) buf, sizeof( buf ) ); // this message in goes with both subsampling - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->canFrames.size(), 8 ); } @@ -807,19 +831,24 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleSubsamplingOfSameSignalUse s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); InspectionMatrixSignalCollectionInfo s3{}; s3.signalID = 5678; s3.sampleBufferSize = 150; s3.minimumSampleIntervalMs = 20; s3.fixedWindowPeriod = 300; + s3.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s3 ); InspectionMatrixSignalCollectionInfo s5{}; s5.signalID = 1111; s5.sampleBufferSize = 10; s5.minimumSampleIntervalMs = 20; s5.fixedWindowPeriod = 300; + s5.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s5 ); + // Condition contains signal + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getTwoSignalsBiggerCondition( s1.signalID, 150.0, s3.signalID, 155.0 ).get(); collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; @@ -829,6 +858,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleSubsamplingOfSameSignalUse s2.sampleBufferSize = 200; s2.minimumSampleIntervalMs = 50; // Different subsampling then signal in other condition s2.fixedWindowPeriod = 150; + s2.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s2 ); InspectionMatrixSignalCollectionInfo s4{}; @@ -836,13 +866,17 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleSubsamplingOfSameSignalUse s4.sampleBufferSize = 130; s4.minimumSampleIntervalMs = 60; s4.fixedWindowPeriod = 250; + s4.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s4 ); InspectionMatrixSignalCollectionInfo s6{}; s6.signalID = 2222; s6.sampleBufferSize = 180; s6.minimumSampleIntervalMs = 50; s6.fixedWindowPeriod = 250; + s6.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s6 ); + // Condition contains signal + collectionSchemes->conditions[1].isStaticCondition = false; collectionSchemes->conditions[1].condition = getTwoSignalsBiggerCondition( s2.signalID, 165.0, s4.signalID, 170.0 ).get(); collectionSchemes->conditions[1].minimumPublishIntervalMs = 10000; @@ -852,23 +886,23 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleSubsamplingOfSameSignalUse for ( int i = 0; i < 100; i++ ) { - engine.addNewSignal( s1.signalID, timestamp + i * 10, i * 2 ); - engine.addNewSignal( s3.signalID, timestamp + i * 10, i * 2 + 1 ); - engine.addNewSignal( s5.signalID, timestamp + i * 10, 55555 ); - engine.addNewSignal( s6.signalID, timestamp + i * 10, 77777 ); + engine.addNewSignal( s1.signalID, timestamp + i * 10, timestamp.monotonicTimeMs + i * 10, i * 2 ); + engine.addNewSignal( s3.signalID, timestamp + i * 10, timestamp.monotonicTimeMs + i * 10, i * 2 + 1 ); + engine.addNewSignal( s5.signalID, timestamp + i * 10, timestamp.monotonicTimeMs + i * 10, 55555 ); + engine.addNewSignal( s6.signalID, timestamp + i * 10, timestamp.monotonicTimeMs + i * 10, 77777 ); } - engine.evaluateConditions( timestamp + 10000 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); using CollectionType = std::shared_ptr; uint32_t waitTimeMs = 0; std::vector collectedDataList; std::shared_ptr collectedData = - engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData; while ( collectedData != nullptr ) { collectedDataList.push_back( collectedData ); - collectedData = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + collectedData = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData; } ASSERT_EQ( collectedDataList.size(), 2 ); @@ -901,13 +935,17 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleFixedWindowsOfSameSignalUs s1.sampleBufferSize = 500; s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); InspectionMatrixSignalCollectionInfo s5{}; s5.signalID = 1111; s5.sampleBufferSize = 500; s5.minimumSampleIntervalMs = 20; s5.fixedWindowPeriod = 0; + s5.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s5 ); + // Condition contains signal + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getLastAvgWindowBiggerCondition( s1.signalID, 150.0 ).get(); collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; @@ -916,6 +954,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleFixedWindowsOfSameSignalUs s2.sampleBufferSize = 500; s2.minimumSampleIntervalMs = 50; // Different subsampling then signal in other condition s2.fixedWindowPeriod = 200; + s2.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s2 ); InspectionMatrixSignalCollectionInfo s6{}; @@ -923,7 +962,10 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleFixedWindowsOfSameSignalUs s6.sampleBufferSize = 500; s6.minimumSampleIntervalMs = 50; s6.fixedWindowPeriod = 250; + s6.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s6 ); + // Condition contains signal + collectionSchemes->conditions[1].isStaticCondition = false; collectionSchemes->conditions[1].condition = getLastAvgWindowBiggerCondition( s2.signalID, 150.0 ).get(); collectionSchemes->conditions[1].minimumPublishIntervalMs = 10000; @@ -932,22 +974,26 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultipleFixedWindowsOfSameSignalUs for ( int i = 0; i < 300; i++ ) { - engine.addNewSignal( s1.signalID, timestamp + 10000 + i * 10, i * 2 ); - engine.addNewSignal( s5.signalID, timestamp + 10000 + i * 10, 55555 ); - engine.addNewSignal( s6.signalID, timestamp + 10000 + i * 10, 77777 ); + engine.addNewSignal( + s1.signalID, timestamp + 10000 + i * 10, timestamp.monotonicTimeMs + 10000 + i * 10, i * 2 ); + engine.addNewSignal( + s5.signalID, timestamp + 10000 + i * 10, timestamp.monotonicTimeMs + 10000 + i * 10, 55555 ); + engine.addNewSignal( + s6.signalID, timestamp + 10000 + i * 10, timestamp.monotonicTimeMs + 10000 + i * 10, 77777 ); } - engine.evaluateConditions( timestamp + 10000 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); using CollectionType = std::shared_ptr; uint32_t waitTimeMs = 0; std::vector collectedDataList; std::shared_ptr collectedData = - engine.collectNextDataToSend( timestamp + 10000 + 100 * 10, waitTimeMs ); + engine.collectNextDataToSend( timestamp + 10000 + 100 * 10, waitTimeMs ).triggeredCollectionSchemeData; while ( collectedData != nullptr ) { collectedDataList.push_back( collectedData ); - collectedData = engine.collectNextDataToSend( timestamp + 10000 + 300 * 10, waitTimeMs ); + collectedData = + engine.collectNextDataToSend( timestamp + 10000 + 300 * 10, waitTimeMs ).triggeredCollectionSchemeData; } ASSERT_EQ( collectedDataList.size(), 2 ); @@ -980,24 +1026,26 @@ TEST_F( CollectionInspectionEngineDoubleTest, Subsampling ) // minimumSampleIntervalMs=10 means faster signals are dropped s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.addNewSignal( s1.signalID, timestamp + 1, 0.2 ); - engine.addNewSignal( s1.signalID, timestamp + 9, 0.3 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp + 1, timestamp.monotonicTimeMs + 1, 0.2 ); + engine.addNewSignal( s1.signalID, timestamp + 9, timestamp.monotonicTimeMs + 9, 0.3 ); // As subsampling is 10 this value should be sampled again - engine.addNewSignal( s1.signalID, timestamp + 10, 0.4 ); - engine.addNewSignal( s1.signalID, timestamp + 40, 0.5 ); + engine.addNewSignal( s1.signalID, timestamp + 10, timestamp.monotonicTimeMs + 10, 0.4 ); + engine.addNewSignal( s1.signalID, timestamp + 40, timestamp.monotonicTimeMs + 40, 0.5 ); - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 3 ); @@ -1007,7 +1055,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, Subsampling ) } // Only valid when sendDataOnlyOncePerCondition is defined true -TEST_F( CollectionInspectionEngineDoubleTest, SendoutEverySignalOnlyOnce ) +TEST_F( CollectionInspectionEngineDoubleTest, SendOutEverySignalOnlyOnce ) { CollectionInspectionEngine engine; InspectionMatrixSignalCollectionInfo s1{}; @@ -1016,36 +1064,38 @@ TEST_F( CollectionInspectionEngineDoubleTest, SendoutEverySignalOnlyOnce ) // if this is not 0 the samples that come too late will be dropped s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // Every 10 seconds send data out collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto res1 = engine.collectNextDataToSend( timestamp, waitTimeMs ); + auto res1 = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_EQ( res1, nullptr ); - engine.evaluateConditions( timestamp + 10000 ); - auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); + auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res2, nullptr ); EXPECT_EQ( res2->signals.size(), 1 ); // Very old element in queue gets pushed - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); - engine.evaluateConditions( timestamp + 20000 ); - auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 20000 ) ); + auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res3, nullptr ); EXPECT_EQ( res3->signals.size(), 1 ); } -TYPED_TEST( CollectionInspectionEngineTest, HearbeatInterval ) +TYPED_TEST( CollectionInspectionEngineTest, HeartbeatInterval ) { CollectionInspectionEngine engine; InspectionMatrixSignalCollectionInfo s1{}; @@ -1058,34 +1108,35 @@ TYPED_TEST( CollectionInspectionEngineTest, HearbeatInterval ) // Every 10 seconds send data out this->collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; + // Condition is static by default this->collectionSchemes->conditions[0].condition = this->getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( this->consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 11 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 11 ); engine.evaluateConditions( timestamp ); uint32_t waitTimeMs = 0; // it should not trigger because less than 10 seconds passed - EXPECT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + EXPECT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); - engine.addNewSignal( s1.signalID, timestamp + 10000, 11 ); - engine.evaluateConditions( timestamp + 10000 ); + engine.addNewSignal( s1.signalID, timestamp + 10000, timestamp.monotonicTimeMs + 10000, 11 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); // Triggers after 10s - EXPECT_NE( engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ), nullptr ); + EXPECT_NE( engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); - engine.addNewSignal( s1.signalID, timestamp + 15000, 11 ); - engine.evaluateConditions( timestamp + 15000 ); + engine.addNewSignal( s1.signalID, timestamp + 15000, timestamp.monotonicTimeMs + 15000, 11 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp + 15000 ) ); // Not triggers after 15s - EXPECT_EQ( engine.collectNextDataToSend( timestamp + 15000, waitTimeMs ), nullptr ); + EXPECT_EQ( engine.collectNextDataToSend( timestamp + 15000, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); - engine.addNewSignal( s1.signalID, timestamp + 20000, 11 ); - engine.evaluateConditions( timestamp + 20000 ); + engine.addNewSignal( s1.signalID, timestamp + 20000, timestamp.monotonicTimeMs + 20000, 11 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 20000 ) ); // Triggers after 20s - EXPECT_NE( engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ), nullptr ); + EXPECT_NE( engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -1100,7 +1151,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RawBufferHandleUsageHints ) s1.sampleBufferSize = 10; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; - s1.signalType = SignalType::RAW_DATA_BUFFER_HANDLE; + s1.signalType = SignalType::COMPLEX_SIGNAL; this->addSignalToCollect( this->collectionSchemes->conditions[0], s1 ); RawData::SignalUpdateConfig signalConfig; @@ -1109,6 +1160,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RawBufferHandleUsageHints ) rawDataManager->updateConfig( { { signalConfig.typeId, signalConfig } } ); this->collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000000; + // Condition is static by default this->collectionSchemes->conditions[0].condition = this->getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -1119,7 +1171,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RawBufferHandleUsageHints ) timestamp++; uint8_t dummyData[] = { 0xDE, 0xAD }; auto handle = rawDataManager->push( dummyData, sizeof( dummyData ), timestamp.systemTimeMs, s1.signalID ); - engine.addNewSignal( s1.signalID, timestamp, handle ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, handle ); } auto statistics = rawDataManager->getStatistics( s1.signalID ); EXPECT_EQ( statistics.numOfSamplesCurrentlyInMemory, 10 ); @@ -1130,12 +1182,65 @@ TEST_F( CollectionInspectionEngineDoubleTest, RawBufferHandleUsageHints ) timestamp++; uint8_t dummyData[] = { 0xDE, 0xAD }; auto handle = rawDataManager->push( dummyData, sizeof( dummyData ), timestamp.systemTimeMs, s1.signalID ); - engine.addNewSignal( s1.signalID, timestamp, handle ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, handle ); } auto statistics2 = rawDataManager->getStatistics( s1.signalID ); EXPECT_EQ( statistics2.numOfSamplesCurrentlyInMemory, 10 ); // EXPECT_EQ( statistics2.overallNumOfSamplesReceived, 20 ); } + +TEST_F( CollectionInspectionEngineDoubleTest, HearbeatIntervalWithVisionSystemData ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 1234; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 10; + s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; + this->addSignalToCollect( this->collectionSchemes->conditions[0], s1 ); + + InspectionMatrixSignalCollectionInfo s2{}; + s2.signalID = 5678; + s2.sampleBufferSize = 10; + s2.minimumSampleIntervalMs = 0; + s2.fixedWindowPeriod = 77777; + s2.signalType = SignalType::COMPLEX_SIGNAL; + this->addSignalToCollect( this->collectionSchemes->conditions[0], s2 ); + + // Every 10 seconds send data out + this->collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; + // Condition is static be default + this->collectionSchemes->conditions[0].condition = this->getAlwaysTrueCondition().get(); + + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( this->consCollectionSchemes, timestamp ); + + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 10 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 9000 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + uint32_t waitTimeMs = 0; + auto collected = engine.collectNextDataToSend( timestamp, waitTimeMs ); + // it should not trigger because less than 10 seconds passed + ASSERT_EQ( collected.triggeredCollectionSchemeData, nullptr ); + ASSERT_EQ( collected.triggeredVisionSystemData, nullptr ); + + engine.addNewSignal( s1.signalID, timestamp + 10000, timestamp.monotonicTimeMs + 10000, 11 ); + engine.addNewSignal( s2.signalID, timestamp + 10000, timestamp.monotonicTimeMs + 10000, 9001 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); + + // Triggers after 10s + collected = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + ASSERT_NE( collected.triggeredCollectionSchemeData, nullptr ); + ASSERT_EQ( collected.triggeredCollectionSchemeData->signals.size(), 2 ); + EXPECT_EQ( collected.triggeredCollectionSchemeData->signals[0].value.value.doubleVal, 11 ); + EXPECT_EQ( collected.triggeredCollectionSchemeData->signals[1].value.value.doubleVal, 10 ); + + ASSERT_NE( collected.triggeredVisionSystemData, nullptr ); + ASSERT_EQ( collected.triggeredVisionSystemData->signals.size(), 2 ); + EXPECT_EQ( collected.triggeredVisionSystemData->signals[0].value.value.uint32Val, 9001 ); + EXPECT_EQ( collected.triggeredVisionSystemData->signals[1].value.value.uint32Val, 9000 ); +} #endif TEST_F( CollectionInspectionEngineDoubleTest, TwoCollectionSchemesWithDifferentNumberOfSamplesToCollect ) @@ -1146,13 +1251,16 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoCollectionSchemesWithDifferentN s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 1235; s2.sampleBufferSize = 30; s2.minimumSampleIntervalMs = 10; s2.fixedWindowPeriod = 77777; + s2.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); addSignalToCollect( collectionSchemes->conditions[0], s2 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); // Collection Scheme 2 collects the same signal with the same minimumSampleIntervalMs @@ -1162,13 +1270,16 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoCollectionSchemesWithDifferentN s3.sampleBufferSize = 500; s3.minimumSampleIntervalMs = s1.minimumSampleIntervalMs; s3.fixedWindowPeriod = 77777; + s3.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s4{}; s4.signalID = s2.signalID; s4.sampleBufferSize = 300; s4.minimumSampleIntervalMs = s1.minimumSampleIntervalMs; s4.fixedWindowPeriod = 77777; + s4.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[1], s3 ); addSignalToCollect( collectionSchemes->conditions[1], s4 ); + // Condition is static by default collectionSchemes->conditions[1].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; @@ -1177,15 +1288,15 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoCollectionSchemesWithDifferentN for ( int i = 0; i < 10000; i++ ) { timestamp++; - engine.addNewSignal( s1.signalID, timestamp, i * i ); - engine.addNewSignal( s2.signalID, timestamp, i + 2 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, i * i ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, i + 2 ); } - engine.evaluateConditions( timestamp ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); - auto collectedData2 = engine.collectNextDataToSend( timestamp, waitTimeMs ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; + auto collectedData2 = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); ASSERT_NE( collectedData, nullptr ); ASSERT_NE( collectedData2, nullptr ); @@ -1205,54 +1316,59 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoSignalsInConditionAndOneSignalT s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; s1.isConditionOnlySignal = true; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 2; s2.sampleBufferSize = 50; s2.minimumSampleIntervalMs = 10; s2.fixedWindowPeriod = 77777; s2.isConditionOnlySignal = true; + s2.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s3{}; s3.signalID = 3; s3.sampleBufferSize = 50; s3.minimumSampleIntervalMs = 0; s3.fixedWindowPeriod = 77777; s3.isConditionOnlySignal = false; + s3.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); addSignalToCollect( collectionSchemes->conditions[0], s2 ); addSignalToCollect( collectionSchemes->conditions[0], s3 ); // The condition is (signalID(1)>-100) && (signalID(2)>-500) + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getTwoSignalsBiggerCondition( s1.signalID, -100.0, s2.signalID, -500.0 ).get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s3.signalID, timestamp, 1000.0 ); - engine.addNewSignal( s3.signalID, timestamp, 2000.0 ); - engine.addNewSignal( s3.signalID, timestamp, 3000.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, 1000.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, 2000.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, 3000.0 ); // Signals for condition are not available yet so collectionScheme should not trigger - engine.evaluateConditions( timestamp ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); timestamp += 1000; - engine.addNewSignal( s1.signalID, timestamp, -90.0 ); - engine.addNewSignal( s2.signalID, timestamp, -1000.0 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -90.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -1000.0 ); // Condition only for first signal is fulfilled (-90 > -100) but second not so boolean and is false - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); timestamp += 1000; - engine.addNewSignal( s2.signalID, timestamp, -480.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -480.0 ); // Condition is fulfilled so it should trigger - engine.evaluateConditions( timestamp ); - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); ASSERT_EQ( collectedData->signals.size(), 3 ); ASSERT_EQ( collectedData->signals[0].signalID, s3.signalID ); @@ -1261,6 +1377,65 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoSignalsInConditionAndOneSignalT ASSERT_EQ( collectedData->triggerTime, timestamp.systemTimeMs ); } +TEST_F( CollectionInspectionEngineDoubleTest, RisingEdgeTrigger ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 1; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 10; + s1.fixedWindowPeriod = 77777; + s1.isConditionOnlySignal = true; + s1.signalType = SignalType::DOUBLE; + s1.isConditionOnlySignal = false; + InspectionMatrixSignalCollectionInfo s2{}; + s2.signalID = 2; + s2.sampleBufferSize = 50; + s2.minimumSampleIntervalMs = 10; + s2.fixedWindowPeriod = 77777; + s2.isConditionOnlySignal = true; + s2.signalType = SignalType::DOUBLE; + addSignalToCollect( collectionSchemes->conditions[0], s1 ); + addSignalToCollect( collectionSchemes->conditions[0], s2 ); + + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; + collectionSchemes->conditions[0].condition = getNotEqualCondition( s1.signalID, s2.signalID ).get(); + collectionSchemes->conditions[0].triggerOnlyOnRisingEdge = true; + + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 1000.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 2000.0 ); + + // Condition evaluates to true but data is not collected (not rising edge) + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + uint32_t waitTimeMs = 0; + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); + + timestamp += 1000; + + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 0.0 ); + + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); + + timestamp += 1000; + + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -480.0 ); + + // Condition is fulfilled so it should trigger (rising edge false->true) + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; + ASSERT_NE( collectedData, nullptr ); + ASSERT_EQ( collectedData->signals.size(), 2 ); + ASSERT_EQ( collectedData->signals[0].signalID, s1.signalID ); + ASSERT_EQ( collectedData->signals[1].signalID, s1.signalID ); + ASSERT_EQ( collectedData->triggerTime, timestamp.systemTimeMs ); +} + // Default is to send data only out once per collectionScheme TYPED_TEST( CollectionInspectionEngineTest, SendOutEverySignalOnlyOncePerCollectionScheme ) { @@ -1275,27 +1450,28 @@ TYPED_TEST( CollectionInspectionEngineTest, SendOutEverySignalOnlyOncePerCollect // Every 10 seconds send data out this->collectionSchemes->conditions[0].minimumPublishIntervalMs = 10000; + // Condition is static by default this->collectionSchemes->conditions[0].condition = this->getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( this->consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 10 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 10 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto res1 = engine.collectNextDataToSend( timestamp + 1, waitTimeMs ); + auto res1 = engine.collectNextDataToSend( timestamp + 1, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_EQ( res1, nullptr ); - engine.evaluateConditions( timestamp + 10000 ); - auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); + auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res2, nullptr ); EXPECT_EQ( res2->signals.size(), 1 ); - engine.addNewSignal( s1.signalID, timestamp + 15000, 10 ); + engine.addNewSignal( s1.signalID, timestamp + 15000, timestamp.monotonicTimeMs + 15000, 10 ); - engine.evaluateConditions( timestamp + 20000 ); - auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 20000 ) ); + auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res3, nullptr ); EXPECT_EQ( res3->signals.size(), 1 ); } @@ -1308,31 +1484,33 @@ TEST_F( CollectionInspectionEngineDoubleTest, SendOutEverySignalNotOnlyOncePerCo s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // Every 10 seconds send data out collectionSchemes->conditions[0].minimumPublishIntervalMs = 5000; + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - auto res1 = engine.collectNextDataToSend( timestamp + 1, waitTimeMs ); + auto res1 = engine.collectNextDataToSend( timestamp + 1, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_EQ( res1, nullptr ); - engine.evaluateConditions( timestamp + 10000 ); - auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 10000 ) ); + auto res2 = engine.collectNextDataToSend( timestamp + 10000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res2, nullptr ); EXPECT_EQ( res2->signals.size(), 1 ); - engine.addNewSignal( s1.signalID, timestamp + 15000, 0.1 ); + engine.addNewSignal( s1.signalID, timestamp + 15000, timestamp.monotonicTimeMs + 15000, 0.1 ); - engine.evaluateConditions( timestamp + 20000 ); - auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 20000 ) ); + auto res3 = engine.collectNextDataToSend( timestamp + 20000, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( res3, nullptr ); EXPECT_EQ( res3->signals.size(), 2 ); } @@ -1345,9 +1523,11 @@ TEST_F( CollectionInspectionEngineDoubleTest, MoreCollectionSchemesThanSupported s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; + s1.signalType = SignalType::DOUBLE; collectionSchemes->conditions.resize( MAX_NUMBER_OF_ACTIVE_CONDITION + 1 ); for ( uint32_t i = 0; i < MAX_NUMBER_OF_ACTIVE_CONDITION; i++ ) { + // Condition is static be default collectionSchemes->conditions[i].condition = getAlwaysTrueCondition().get(); addSignalToCollect( collectionSchemes->conditions[i], s1 ); } @@ -1356,15 +1536,15 @@ TEST_F( CollectionInspectionEngineDoubleTest, MoreCollectionSchemesThanSupported engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); uint32_t waitTimeMs = 0; - engine.addNewSignal( s1.signalID, timestamp, 0.1 ); - engine.evaluateConditions( timestamp ); - engine.collectNextDataToSend( timestamp, waitTimeMs ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.1 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; timestamp += 10000; - engine.evaluateConditions( timestamp ); - engine.collectNextDataToSend( timestamp, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; timestamp += 10000; - engine.evaluateConditions( timestamp ); - engine.collectNextDataToSend( timestamp, waitTimeMs ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; } TYPED_TEST( CollectionInspectionEngineTest, CollectWithAfterTime ) @@ -1376,12 +1556,14 @@ TYPED_TEST( CollectionInspectionEngineTest, CollectWithAfterTime ) s1.minimumSampleIntervalMs = 10; s1.fixedWindowPeriod = 77777; s1.isConditionOnlySignal = true; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 2; s2.sampleBufferSize = 50; s2.minimumSampleIntervalMs = 10; s2.fixedWindowPeriod = 77777; s2.isConditionOnlySignal = true; + s2.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s3{}; s3.signalID = 3; s3.sampleBufferSize = 3; @@ -1394,6 +1576,8 @@ TYPED_TEST( CollectionInspectionEngineTest, CollectWithAfterTime ) this->addSignalToCollect( this->collectionSchemes->conditions[0], s3 ); // The condition is (signalID(1)>-100) && (signalID(2)>-500) + // Condition contains signals + this->collectionSchemes->conditions[0].isStaticCondition = false; this->collectionSchemes->conditions[0].condition = this->getTwoSignalsBiggerCondition( s1.signalID, -100.0, s2.signalID, -500.0 ).get(); @@ -1404,56 +1588,56 @@ TYPED_TEST( CollectionInspectionEngineTest, CollectWithAfterTime ) engine.onChangeInspectionMatrix( this->consCollectionSchemes, timestamp ); TypeParam val1 = 10; - engine.addNewSignal( s3.signalID, timestamp, val1 ); - engine.addNewSignal( s1.signalID, timestamp, -90.0 ); - engine.addNewSignal( s2.signalID, timestamp, -1000.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, val1 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -90.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -1000.0 ); // Condition not fulfilled so should not trigger - engine.evaluateConditions( timestamp ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); EXPECT_GE( waitTimeMs, 2000 ); timestamp += 1000; TimePoint timestamp0 = timestamp; TypeParam val2 = 20; - engine.addNewSignal( s3.signalID, timestamp, val2 ); - engine.addNewSignal( s1.signalID, timestamp, -90.0 ); - engine.addNewSignal( s2.signalID, timestamp, -480.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, val2 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -90.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -480.0 ); // Condition fulfilled so should trigger but afterTime not over yet - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); EXPECT_EQ( waitTimeMs, 2000 ); timestamp += 1000; TimePoint timestamp1 = timestamp; TypeParam val3 = 30; - engine.addNewSignal( s3.signalID, timestamp, val3 ); - engine.addNewSignal( s1.signalID, timestamp, -95.0 ); - engine.addNewSignal( s2.signalID, timestamp, -485.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, val3 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -95.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -485.0 ); // Condition still fulfilled but already triggered. After time still not over - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); EXPECT_EQ( waitTimeMs, 1000 ); timestamp += 500; TimePoint timestamp2 = timestamp; TypeParam val4 = 40; - engine.addNewSignal( s3.signalID, timestamp, val4 ); - engine.addNewSignal( s1.signalID, timestamp, -9000.0 ); - engine.addNewSignal( s2.signalID, timestamp, -9000.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, val4 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -9000.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -9000.0 ); // Condition not fulfilled anymore but still waiting for afterTime - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); timestamp += 500; TimePoint timestamp3 = timestamp; TypeParam val5 = 50; - engine.addNewSignal( s3.signalID, timestamp, val5 ); - engine.addNewSignal( s1.signalID, timestamp, -9100.0 ); - engine.addNewSignal( s2.signalID, timestamp, -9100.0 ); + engine.addNewSignal( s3.signalID, timestamp, timestamp.monotonicTimeMs, val5 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -9100.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, -9100.0 ); // Condition not fulfilled. After Time is over so data should be sent out - engine.evaluateConditions( timestamp ); - auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; ASSERT_NE( collectedData, nullptr ); // sampleBufferSize of the only collected signal s3 is 3 @@ -1476,9 +1660,12 @@ TEST_F( CollectionInspectionEngineDoubleTest, AvgWindowCondition ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 300000; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // function is: LAST_FIXED_WINDOW_AVG(SignalID(1234)) > -50.0 + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getLastAvgWindowBiggerCondition( s1.signalID, -50.0 ).get(); TimePoint timestamp = { 160000000, 100 }; @@ -1492,9 +1679,9 @@ TEST_F( CollectionInspectionEngineDoubleTest, AvgWindowCondition ) { timestamp++; currentValue += increasePerSample; - engine.addNewSignal( s1.signalID, timestamp, currentValue ); - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, currentValue ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } currentValue = -80.0f; @@ -1504,13 +1691,13 @@ TEST_F( CollectionInspectionEngineDoubleTest, AvgWindowCondition ) { timestamp++; currentValue += increasePerSample; - engine.addNewSignal( s1.signalID, timestamp, currentValue ); - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, currentValue ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } // avg -30 is bigger than -50 so condition fulfilled - engine.evaluateConditions( timestamp + 2 ); - ASSERT_NE( engine.collectNextDataToSend( timestamp + 2, waitTimeMs ), nullptr ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 2 ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp + 2, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, PrevLastAvgWindowCondition ) @@ -1522,9 +1709,12 @@ TEST_F( CollectionInspectionEngineDoubleTest, PrevLastAvgWindowCondition ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 300000; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // function is: PREV_LAST_FIXED_WINDOW_AVG(SignalID(1234)) > -50.0 + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getPrevLastAvgWindowBiggerCondition( s1.signalID, -50.0 ).get(); TimePoint timestamp = { 160000000, 100 }; @@ -1535,16 +1725,16 @@ TEST_F( CollectionInspectionEngineDoubleTest, PrevLastAvgWindowCondition ) for ( uint32_t i = 0; i < s1.fixedWindowPeriod; i++ ) { timestamp++; - engine.addNewSignal( s1.signalID, timestamp, 0.0 ); - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } // No samples arrive for two window2: timestamp += s1.fixedWindowPeriod * 2; // One more arrives, prev last average is still 0 which is > -50 - engine.addNewSignal( s1.signalID, timestamp, -100.0 ); - engine.evaluateConditions( timestamp ); - ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -100.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, MultiWindowCondition ) @@ -1556,30 +1746,33 @@ TEST_F( CollectionInspectionEngineDoubleTest, MultiWindowCondition ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // function is: (((LAST_FIXED_WINDOW_MAX(SignalID(1234)) - PREV_LAST_FIXED_WINDOW_MAX(SignalID(1234))) // + PREV_LAST_FIXED_WINDOW_MAX(SignalID(1234))) // < (LAST_FIXED_WINDOW_MAX(SignalID(1234)) * PREV_LAST_FIXED_WINDOW_MAX(SignalID(1234)))) // || (LAST_FIXED_WINDOW_MIN(SignalID(1234)) == PREV_LAST_FIXED_WINDOW_MIN(SignalID(1234))) + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getMultiFixedWindowCondition( s1.signalID ).get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); - engine.addNewSignal( s1.signalID, timestamp, -95.0 ); - engine.addNewSignal( s1.signalID, timestamp + 50, 100.0 ); - engine.addNewSignal( s1.signalID, timestamp + 70, 110.0 ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, -95.0 ); + engine.addNewSignal( s1.signalID, timestamp + 50, timestamp.monotonicTimeMs + 50, 100.0 ); + engine.addNewSignal( s1.signalID, timestamp + 70, timestamp.monotonicTimeMs + 70, 110.0 ); // Condition still fulfilled but already triggered. After time still not over - engine.evaluateConditions( timestamp + 70 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp + 70 ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp + 70, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp + 70, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); - engine.addNewSignal( s1.signalID, timestamp + 100, -205.0 ); - engine.addNewSignal( s1.signalID, timestamp + 150, -300.0 ); - engine.addNewSignal( s1.signalID, timestamp + 200, +30.0 ); - engine.evaluateConditions( timestamp + 200 ); - ASSERT_NE( engine.collectNextDataToSend( timestamp + 200, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp + 100, timestamp.monotonicTimeMs + 100, -205.0 ); + engine.addNewSignal( s1.signalID, timestamp + 150, timestamp.monotonicTimeMs + 150, -300.0 ); + engine.addNewSignal( s1.signalID, timestamp + 200, timestamp.monotonicTimeMs + 200, +30.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp + 200 ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp + 200, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, TestNotEqualOperator ) @@ -1590,38 +1783,131 @@ TEST_F( CollectionInspectionEngineDoubleTest, TestNotEqualOperator ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 456; s2.sampleBufferSize = 50; s2.minimumSampleIntervalMs = 0; s2.fixedWindowPeriod = 100; + s2.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); addSignalToCollect( collectionSchemes->conditions[0], s2 ); // function is: !(!(SignalID(123) <= 0.001) && !((SignalID(123) / SignalID(456)) >= 0.5)) + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getNotEqualCondition( s1.signalID, s2.signalID ).get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // Expression should be false because they are equal - engine.addNewSignal( s1.signalID, timestamp, 100.0 ); - engine.addNewSignal( s2.signalID, timestamp, 100.0 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); // Expression should be true: because they are not equal timestamp++; - engine.addNewSignal( s1.signalID, timestamp, 50 ); - engine.evaluateConditions( timestamp ); - ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 50 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); // Expression should be false because they are equal again timestamp++; - engine.addNewSignal( s2.signalID, timestamp, 50.0 ); - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 50.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); +} + +TEST_F( CollectionInspectionEngineDoubleTest, TestBoolToDoubleImplicitCast ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 123; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 0; + s1.fixedWindowPeriod = 100; + addSignalToCollect( collectionSchemes->conditions[0], s1 ); + + // expression is: True + 1.0 == 2.0 + expressionNodes.push_back( std::make_shared() ); + auto nodeLeft = expressionNodes.back(); + nodeLeft->nodeType = ExpressionNodeType::BOOLEAN; + nodeLeft->booleanValue = true; + + expressionNodes.push_back( std::make_shared() ); + auto nodeRight = expressionNodes.back(); + nodeRight->nodeType = ExpressionNodeType::FLOAT; + nodeRight->floatingValue = 1.0; + + expressionNodes.push_back( std::make_shared() ); + auto nodeOperation = expressionNodes.back(); + nodeOperation->nodeType = ExpressionNodeType::OPERATOR_ARITHMETIC_PLUS; + nodeOperation->left = nodeLeft.get(); + nodeOperation->right = nodeRight.get(); + + expressionNodes.push_back( std::make_shared() ); + auto nodeRight2 = expressionNodes.back(); + nodeRight2->nodeType = ExpressionNodeType::FLOAT; + nodeRight2->floatingValue = 2.0; + + expressionNodes.push_back( std::make_shared() ); + auto nodeOperation2 = expressionNodes.back(); + nodeOperation2->nodeType = ExpressionNodeType::OPERATOR_EQUAL; + nodeOperation2->left = nodeOperation.get(); + nodeOperation2->right = nodeRight2.get(); + // Condition is static be default + collectionSchemes->conditions[0].condition = nodeOperation2.get(); + + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + // Expression should be true + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + uint32_t waitTimeMs = 0; + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); +} + +TEST_F( CollectionInspectionEngineDoubleTest, TestDoubleToBoolImplicitCast ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 123; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 0; + s1.fixedWindowPeriod = 100; + addSignalToCollect( collectionSchemes->conditions[0], s1 ); + + // expression is: 42.0 && True + expressionNodes.push_back( std::make_shared() ); + auto nodeLeft = expressionNodes.back(); + nodeLeft->nodeType = ExpressionNodeType::FLOAT; + nodeLeft->floatingValue = 42.0; + + expressionNodes.push_back( std::make_shared() ); + auto nodeRight = expressionNodes.back(); + nodeRight->nodeType = ExpressionNodeType::BOOLEAN; + nodeRight->booleanValue = true; + + expressionNodes.push_back( std::make_shared() ); + auto nodeOperation = expressionNodes.back(); + nodeOperation->nodeType = ExpressionNodeType::OPERATOR_LOGICAL_AND; + nodeOperation->left = nodeLeft.get(); + nodeOperation->right = nodeRight.get(); + // Condition is static be default + collectionSchemes->conditions[0].condition = nodeOperation.get(); + + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + // Expression should be true + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + uint32_t waitTimeMs = 0; + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, TwoSignalsRatioCondition ) @@ -1632,15 +1918,19 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoSignalsRatioCondition ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; InspectionMatrixSignalCollectionInfo s2{}; s2.signalID = 456; s2.sampleBufferSize = 50; s2.minimumSampleIntervalMs = 0; s2.fixedWindowPeriod = 100; + s2.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); addSignalToCollect( collectionSchemes->conditions[0], s2 ); // function is: !(!(SignalID(123) <= 0.001) && !((SignalID(123) / SignalID(456)) >= 0.5)) + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getTwoSignalsRatioCondition( s1.signalID, 0.001, s2.signalID, 0.5 ).get(); @@ -1648,29 +1938,29 @@ TEST_F( CollectionInspectionEngineDoubleTest, TwoSignalsRatioCondition ) engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // Expression should be false: (1.0 <= 0.001) || (1.0 / 100.0) >= 0.5) - engine.addNewSignal( s1.signalID, timestamp, 1.0 ); - engine.addNewSignal( s2.signalID, timestamp, 100.0 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 1.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); // Expression should be true: (0.001 <= 0.001) || (0.001 / 100.0) >= 0.5) timestamp++; - engine.addNewSignal( s1.signalID, timestamp, 0.001 ); - engine.evaluateConditions( timestamp ); - ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 0.001 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); // Expression should be false: (1.0 <= 0.001) || (1.0 / 100.0) >= 0.5) timestamp++; - engine.addNewSignal( s1.signalID, timestamp, 1.0 ); - engine.evaluateConditions( timestamp ); - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 1.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); // Expression should be true: (50.0 <= 0.001) || (50.0 / 100.0) >= 0.5) timestamp++; - engine.addNewSignal( s1.signalID, timestamp, 50.0 ); - engine.evaluateConditions( timestamp ); - ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 50.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + ASSERT_NE( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, UnknownExpressionNode ) @@ -1681,19 +1971,22 @@ TEST_F( CollectionInspectionEngineDoubleTest, UnknownExpressionNode ) s1.sampleBufferSize = 50; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 100; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); // function is: (SignalID(123) <= Unknown) + // Condition is not static + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getUnknownCondition( s1.signalID ).get(); TimePoint timestamp = { 160000000, 100 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); // Expression should be false: (1.0 <= Unknown) - engine.addNewSignal( s1.signalID, timestamp, 1.0 ); - engine.evaluateConditions( timestamp ); + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 1.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); uint32_t waitTimeMs = 0; - ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ), nullptr ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); } TEST_F( CollectionInspectionEngineDoubleTest, RequestTooMuchMemorySignals ) @@ -1705,7 +1998,9 @@ TEST_F( CollectionInspectionEngineDoubleTest, RequestTooMuchMemorySignals ) s1.sampleBufferSize = 1000000; s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 300000; + s1.signalType = SignalType::DOUBLE; addSignalToCollect( collectionSchemes->conditions[0], s1 ); + // Condition is static by default collectionSchemes->conditions[1].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 0, 0 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); @@ -1720,6 +2015,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RequestTooMuchMemoryFrames ) c1.sampleBufferSize = 1000000; c1.minimumSampleIntervalMs = 0; collectionSchemes->conditions[0].canFrames.push_back( c1 ); + // Condition is static by default collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); TimePoint timestamp = { 0, 0 }; engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); @@ -1745,6 +2041,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RandomDataTest ) ConditionWithCollectedData collectionScheme; collectionSchemes->conditions.resize( NUMBER_OF_COLLECTION_SCHEMES ); + // Condition is static by default collectionSchemes->conditions[i].condition = getAlwaysTrueCondition().get(); } @@ -1774,6 +2071,7 @@ TEST_F( CollectionInspectionEngineDoubleTest, RandomDataTest ) s1.sampleBufferSize = 10; s1.minimumSampleIntervalMs = 1; s1.fixedWindowPeriod = windowSize; + s1.signalType = SignalType::DOUBLE; for ( int j = 0; j < normalDistribution( gen ); j++ ) { auto &collectionScheme = collectionSchemes->conditions[uniformDistribution( gen )]; @@ -1782,6 +2080,8 @@ TEST_F( CollectionInspectionEngineDoubleTest, RandomDataTest ) // exactly two signals are added now so add them to condition: if ( collectionScheme.signals.size() == 2 ) { + // Condition contains signals + collectionScheme.isStaticCondition = false; if ( uniformDistribution( gen ) < NUMBER_OF_COLLECTION_SCHEMES / 2 ) { collectionScheme.condition = getLastAvgWindowBiggerCondition( s1.signalID, 15 ).get(); @@ -1802,7 +2102,21 @@ TEST_F( CollectionInspectionEngineDoubleTest, RandomDataTest ) TimePoint START_TIMESTAMP = { 160000000, 100 }; TimePoint timestamp = START_TIMESTAMP; - engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + auto originalLogLevel = Aws::IoTFleetWise::gSystemWideLogLevel; + try + { + // Temporarily change the log level since we have too many signals, which would make the test + // output too noisy with Trace level. + Aws::IoTFleetWise::gSystemWideLogLevel = Aws::IoTFleetWise::LogLevel::Info; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + } + catch ( ... ) + { + Aws::IoTFleetWise::gSystemWideLogLevel = originalLogLevel; + throw; + } + Aws::IoTFleetWise::gSystemWideLogLevel = originalLogLevel; const int TIME_TO_SIMULATE_IN_MS = 5000; const int SIGNALS_PER_MS = 20; @@ -1826,12 +2140,13 @@ TEST_F( CollectionInspectionEngineDoubleTest, RandomDataTest ) counter = 0; tickIncreased = true; } - engine.addNewSignal( signalIDGenerator( gen2 ), timestamp, normalDistribution( gen ) ); + engine.addNewSignal( + signalIDGenerator( gen2 ), timestamp, timestamp.monotonicTimeMs, normalDistribution( gen ) ); if ( tickIncreased ) { uint32_t waitTimeMs = 0; engine.evaluateConditions( timestamp ); - while ( engine.collectNextDataToSend( timestamp, waitTimeMs ) != nullptr ) + while ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData != nullptr ) { dataCollected++; } @@ -1851,5 +2166,70 @@ TEST_F( CollectionInspectionEngineDoubleTest, NoCollectionSchemes ) ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); } +// Test to assert that a signal buffer is not allocated for a signal not known to DM +TEST_F( CollectionInspectionEngineDoubleTest, UnknownSignalNoSignalBuffer ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 1234; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 10; + s1.signalType = SignalType::UNKNOWN; + addSignalToCollect( collectionSchemes->conditions[0], s1 ); + + // Condition is static by default + collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); + collectionSchemes->conditions[0].metadata.collectionSchemeID = "Test Campaign"; + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 100.0 ); + ASSERT_TRUE( engine.evaluateConditions( timestamp ) ); + // Condition will evaluate to true but no data will be collected + uint32_t waitTimeMs = 0; + auto collectedData = engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData; + ASSERT_EQ( collectedData->signals.size(), 0 ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp + 5000, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); +} + +TEST_F( CollectionInspectionEngineDoubleTest, UnknownSignalInExpression ) +{ + CollectionInspectionEngine engine; + InspectionMatrixSignalCollectionInfo s1{}; + s1.signalID = 123; + s1.sampleBufferSize = 50; + s1.minimumSampleIntervalMs = 10; + s1.signalType = SignalType::UNKNOWN; + addSignalToCollect( collectionSchemes->conditions[0], s1 ); + + InspectionMatrixSignalCollectionInfo s2{}; + s2.signalID = 456; + s2.sampleBufferSize = 50; + s2.minimumSampleIntervalMs = 10; + s2.signalType = SignalType::DOUBLE; + addSignalToCollect( collectionSchemes->conditions[0], s2 ); + + collectionSchemes->conditions[0].isStaticCondition = false; + collectionSchemes->conditions[0].condition = getNotEqualCondition( s1.signalID, s2.signalID ).get(); + collectionSchemes->conditions[0].minimumPublishIntervalMs = 5000; + collectionSchemes->conditions[0].metadata.collectionSchemeID = "Test Campaign"; + + TimePoint timestamp = { 160000000, 100 }; + engine.onChangeInspectionMatrix( consCollectionSchemes, timestamp ); + + // Condition evaluates to false + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 90.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 90.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + uint32_t waitTimeMs = 0; + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); + + // Condition still evaluates to false after time increment + timestamp += 5000; + engine.addNewSignal( s1.signalID, timestamp, timestamp.monotonicTimeMs, 110.0 ); + engine.addNewSignal( s2.signalID, timestamp, timestamp.monotonicTimeMs, 110.0 ); + ASSERT_FALSE( engine.evaluateConditions( timestamp ) ); + ASSERT_EQ( engine.collectNextDataToSend( timestamp, waitTimeMs ).triggeredCollectionSchemeData, nullptr ); +} } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/CollectionInspectionWorkerThreadTest.cpp b/test/unit/CollectionInspectionWorkerThreadTest.cpp index 1e4988f4..b9c2b937 100644 --- a/test/unit/CollectionInspectionWorkerThreadTest.cpp +++ b/test/unit/CollectionInspectionWorkerThreadTest.cpp @@ -6,8 +6,11 @@ #include "Clock.h" #include "ClockHandler.h" #include "CollectionInspectionAPITypes.h" +#include "CollectionInspectionEngine.h" +#include "DataSenderTypes.h" #include "ICollectionScheme.h" #include "OBDDataTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "TimeTypes.h" #include "WaitUntil.h" @@ -31,9 +34,16 @@ class CollectionInspectionWorkerThreadTest : public ::testing::Test std::shared_ptr collectionSchemes; std::shared_ptr consCollectionSchemes; std::vector> expressionNodes; - SignalBufferPtr signalBufferPtr; - std::shared_ptr fClock = ClockHandler::getClock(); - std::shared_ptr outputCollectedData; + SignalBufferPtr signalBuffer; + std::shared_ptr mClock = ClockHandler::getClock(); + std::shared_ptr outputCollectedData; + void + initAndStartWorker( CollectionInspectionWorkerThread &worker ) + { + bool res = worker.init( signalBuffer, outputCollectedData, 1000, nullptr ); + ASSERT_TRUE( res ); + ASSERT_TRUE( worker.start() ); + } std::shared_ptr getAlwaysTrueCondition() @@ -76,6 +86,16 @@ class CollectionInspectionWorkerThreadTest : public ::testing::Test return ( expressionNodes.back() ); } + template + bool + popCollectedData( std::shared_ptr &collectedData ) + { + std::shared_ptr senderData; + auto succeeded = outputCollectedData->pop( senderData ); + collectedData = std::dynamic_pointer_cast( senderData ); + return succeeded; + } + void SetUp() override { @@ -85,9 +105,9 @@ class CollectionInspectionWorkerThreadTest : public ::testing::Test collectionSchemes->conditions[0].condition = getAlwaysFalseCondition().get(); collectionSchemes->conditions[1].condition = getAlwaysFalseCondition().get(); - signalBufferPtr.reset( new SignalBuffer( 1000 ) ); + signalBuffer.reset( new SignalBuffer( 1000, "Signal Buffer" ) ); // Init the output buffer - outputCollectedData = std::make_shared( 3 ); + outputCollectedData = std::make_shared( 3, "Collected Data" ); } void @@ -98,9 +118,9 @@ class CollectionInspectionWorkerThreadTest : public ::testing::Test TEST_F( CollectionInspectionWorkerThreadTest, CollectBurstWithoutSubsampling ) { - CollectionInspectionWorkerThread worker; - ASSERT_TRUE( worker.init( signalBufferPtr, outputCollectedData, 1000 ) ); - ASSERT_TRUE( worker.start() ); + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread worker( engine ); + initAndStartWorker( worker ); // minimumSampleIntervalMs=0 means no subsampling InspectionMatrixSignalCollectionInfo s1{}; s1.signalID = 1234; @@ -133,47 +153,49 @@ TEST_F( CollectionInspectionWorkerThreadTest, CollectBurstWithoutSubsampling ) collectionSchemes->conditions[0].signals.push_back( s1 ); collectionSchemes->conditions[0].signals.push_back( s2 ); collectionSchemes->conditions[0].signals.push_back( s3 ); + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getSignalsBiggerCondition( s1.signalID, 1 ).get(); worker.onChangeInspectionMatrix( consCollectionSchemes ); - Timestamp timestamp = fClock->systemTimeSinceEpochMs(); + Timestamp timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 0.1 ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 0.1, SignalType::DOUBLE ) ); collectedSignalsGroup.push_back( CollectedSignal( s3.signalID, timestamp, 0, s3.signalType ) ); collectedSignalsGroup.push_back( CollectedSignal( s2.signalID, timestamp, 10, s2.signalType ) ); std::array buf1 = { 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; - signalBufferPtr->push( CollectedDataFrame( + signalBuffer->push( CollectedDataFrame( collectedSignalsGroup, std::make_shared( c1.frameID, c1.channelID, timestamp, buf1, sizeof( buf1 ) ) ) ); collectedSignalsGroup.clear(); - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 1, 0.2 ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 1, 0.2, SignalType::DOUBLE ) ); collectedSignalsGroup.push_back( CollectedSignal( s3.signalID, timestamp + 1, 1, s3.signalType ) ); collectedSignalsGroup.push_back( CollectedSignal( s2.signalID, timestamp + 1, 15, s2.signalType ) ); std::array buf2 = { 0xBA, 0xAD, 0xAF, 0xFE, 0x0, 0x0, 0x0, 0x0 }; - signalBufferPtr->push( CollectedDataFrame( + signalBuffer->push( CollectedDataFrame( collectedSignalsGroup, std::make_shared( c1.frameID, c1.channelID, timestamp, buf2, sizeof( buf2 ) ) ) ); collectedSignalsGroup.clear(); - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 2, 1.5 ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 2, 1.5, SignalType::DOUBLE ) ); collectedSignalsGroup.push_back( CollectedSignal( s2.signalID, timestamp + 2, 20, s2.signalType ) ); collectedSignalsGroup.push_back( CollectedSignal( s3.signalID, timestamp + 2, 0, s3.signalType ) ); std::array buf3 = { 0xCA, 0xFE, 0xF0, 0x0D, 0x0, 0x0, 0x0, 0x0 }; - signalBufferPtr->push( CollectedDataFrame( + signalBuffer->push( CollectedDataFrame( collectedSignalsGroup, std::make_shared( c1.frameID, c1.channelID, timestamp, buf3, sizeof( buf3 ) ) ) ); worker.onNewDataAvailable(); std::shared_ptr collectedData; - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); ASSERT_EQ( collectedData->signals.size(), 9 ); @@ -195,7 +217,7 @@ TEST_F( CollectionInspectionWorkerThreadTest, CollectBurstWithoutSubsampling ) EXPECT_EQ( collectedData->canFrames[1].data, buf2 ); EXPECT_EQ( collectedData->canFrames[2].data, buf1 ); - ASSERT_FALSE( outputCollectedData->pop( collectedData ) ); + ASSERT_FALSE( popCollectedData( collectedData ) ); // Check changing the inspection matrix when already running: worker.onChangeInspectionMatrix( consCollectionSchemes ); @@ -206,9 +228,9 @@ TEST_F( CollectionInspectionWorkerThreadTest, CollectBurstWithoutSubsampling ) TEST_F( CollectionInspectionWorkerThreadTest, CollectionQueueFull ) { - CollectionInspectionWorkerThread worker; - ASSERT_TRUE( worker.init( signalBufferPtr, outputCollectedData, 1000 ) ); - ASSERT_TRUE( worker.start() ); + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread worker( engine ); + initAndStartWorker( worker ); // minimumSampleIntervalMs=0 means no subsampling InspectionMatrixSignalCollectionInfo s1{}; s1.signalID = 1234; @@ -216,6 +238,7 @@ TEST_F( CollectionInspectionWorkerThreadTest, CollectionQueueFull ) s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; s1.isConditionOnlySignal = false; + s1.signalType = SignalType::DOUBLE; collectionSchemes->conditions[0].signals.push_back( s1 ); collectionSchemes->conditions[0].condition = getAlwaysTrueCondition().get(); collectionSchemes->conditions[1].signals.push_back( s1 ); @@ -225,26 +248,26 @@ TEST_F( CollectionInspectionWorkerThreadTest, CollectionQueueFull ) collectionSchemes->conditions[3].signals.push_back( s1 ); collectionSchemes->conditions[3].condition = getAlwaysTrueCondition().get(); worker.onChangeInspectionMatrix( consCollectionSchemes ); - Timestamp timestamp = fClock->systemTimeSinceEpochMs(); + Timestamp timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 1 ) ); - signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 1, SignalType::DOUBLE ) ); + signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ); worker.onNewDataAvailable(); std::shared_ptr collectedData; - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); - ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); - ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); - ASSERT_FALSE( outputCollectedData->pop( collectedData ) ); + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); + ASSERT_TRUE( popCollectedData( collectedData ) ); + ASSERT_TRUE( popCollectedData( collectedData ) ); + ASSERT_FALSE( popCollectedData( collectedData ) ); worker.stop(); } TEST_F( CollectionInspectionWorkerThreadTest, ConsumeDataWithoutNotify ) { - CollectionInspectionWorkerThread worker; - ASSERT_TRUE( worker.init( signalBufferPtr, outputCollectedData, 1000 ) ); - ASSERT_TRUE( worker.start() ); + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread worker( engine ); + initAndStartWorker( worker ); // minimumSampleIntervalMs=0 means no subsampling InspectionMatrixSignalCollectionInfo s1{}; s1.signalID = 1234; @@ -252,29 +275,31 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeDataWithoutNotify ) s1.minimumSampleIntervalMs = 0; s1.fixedWindowPeriod = 77777; s1.isConditionOnlySignal = false; + s1.signalType = SignalType::DOUBLE; collectionSchemes->conditions[0].signals.push_back( s1 ); + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getSignalsBiggerCondition( s1.signalID, 1 ).get(); collectionSchemes->conditions[0].triggerOnlyOnRisingEdge = true; worker.onChangeInspectionMatrix( consCollectionSchemes ); - Timestamp timestamp = fClock->systemTimeSinceEpochMs(); + Timestamp timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 0.1 ) ); - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp, 0.1, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ) ); collectedSignalsGroup.clear(); - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 2, 0.2 ) ); - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 2, 0.2, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ) ); collectedSignalsGroup.clear(); - collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 3, 1.5 ) ); - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ); + collectedSignalsGroup.push_back( CollectedSignal( s1.signalID, timestamp + 3, 1.5, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ) ); collectedSignalsGroup.clear(); - std::shared_ptr collectedData; - // After one second even without notifying data in the queue should be consumed - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); + std::shared_ptr collectedData; + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); ASSERT_EQ( collectedData->signals.size(), 3U ); @@ -282,7 +307,7 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeDataWithoutNotify ) EXPECT_EQ( collectedData->signals[1].value.value.doubleVal, 0.2 ); EXPECT_EQ( collectedData->signals[2].value.value.doubleVal, 0.1 ); - DELAY_ASSERT_FALSE( outputCollectedData->pop( collectedData ) ); + DELAY_ASSERT_FALSE( popCollectedData( collectedData ) ); worker.stop(); } @@ -292,18 +317,18 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeDataWithoutNotify ) */ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeHasEnabledDTCs ) { - CollectionInspectionWorkerThread inspectionWorker; - ASSERT_TRUE( inspectionWorker.init( signalBufferPtr, outputCollectedData, 1000 ) ); - ASSERT_TRUE( inspectionWorker.start() ); + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread inspectionWorker( engine ); + initAndStartWorker( inspectionWorker ); // Test case 1 : Create a set of DTCs and make sure they are available in the collected data DTCInfo dtcInfo; dtcInfo.mDTCCodes.emplace_back( "P0143" ); dtcInfo.mDTCCodes.emplace_back( "C0196" ); dtcInfo.mSID = SID::STORED_DTC; - dtcInfo.receiveTime = fClock->systemTimeSinceEpochMs(); + dtcInfo.receiveTime = mClock->systemTimeSinceEpochMs(); ASSERT_TRUE( dtcInfo.hasItems() ); // Push the DTCs to the buffer - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); // Prepare a condition to evaluate and expect the DTCs to be collected. InspectionMatrixSignalCollectionInfo signal{}; signal.signalID = 1234; @@ -311,25 +336,28 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeH signal.minimumSampleIntervalMs = 0; signal.fixedWindowPeriod = 77777; signal.isConditionOnlySignal = false; + signal.signalType = SignalType::DOUBLE; collectionSchemes->conditions[0].signals.push_back( signal ); + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getSignalsBiggerCondition( signal.signalID, 1 ).get(); collectionSchemes->conditions[0].triggerOnlyOnRisingEdge = true; // Make sure that DTCs should be collected collectionSchemes->conditions[0].includeActiveDtcs = true; inspectionWorker.onChangeInspectionMatrix( consCollectionSchemes ); - Timestamp timestamp = fClock->systemTimeSinceEpochMs(); + Timestamp timestamp = mClock->systemTimeSinceEpochMs(); CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1 ) ); - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp + 1, 0.2 ) ); - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp + 2, 1.5 ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp + 1, 0.2, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp + 2, 1.5, SignalType::DOUBLE ) ); // Push the signals so that the condition is met - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ) ); std::shared_ptr collectedData; // Expect the data to be collected and has the DTCs - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); ASSERT_TRUE( collectedData->mDTCInfo.hasItems() ); ASSERT_EQ( collectedData->mDTCInfo.mSID, SID::STORED_DTC ); ASSERT_EQ( collectedData->mDTCInfo.mDTCCodes[0], "P0143" ); @@ -340,27 +368,27 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeH dtcInfo.mDTCCodes.emplace_back( "B0148" ); dtcInfo.mDTCCodes.emplace_back( "U0148" ); dtcInfo.mSID = SID::STORED_DTC; - dtcInfo.receiveTime = fClock->systemTimeSinceEpochMs(); + dtcInfo.receiveTime = mClock->systemTimeSinceEpochMs(); // Push the DTCs to the buffer - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); - timestamp = fClock->systemTimeSinceEpochMs(); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); + timestamp = mClock->systemTimeSinceEpochMs(); // Push the signals so that the condition is met CollectedSignalsGroup collectedSignalGroup; - collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1 ) ); - ASSERT_TRUE( signalBufferPtr->push( collectedSignalGroup ) ); + collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( collectedSignalGroup ) ); collectedSignalGroup.clear(); - collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp + 1, 0.2 ) ); - ASSERT_TRUE( signalBufferPtr->push( collectedSignalGroup ) ); + collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp + 1, 0.2, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( collectedSignalGroup ) ); collectedSignalGroup.clear(); - collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp + 2, 1.5 ) ); - ASSERT_TRUE( signalBufferPtr->push( collectedSignalGroup ) ); + collectedSignalGroup.push_back( CollectedSignal( signal.signalID, timestamp + 2, 1.5, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( collectedSignalGroup ) ); // Expect the data to be collected and has the DTCs - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); ASSERT_TRUE( collectedData->mDTCInfo.hasItems() ); ASSERT_EQ( collectedData->mDTCInfo.mSID, SID::STORED_DTC ); ASSERT_EQ( collectedData->mDTCInfo.mDTCCodes[0], "B0148" ); @@ -373,18 +401,18 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeH // then the expected output should NOT include them TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeHasDisabledDTCs ) { - CollectionInspectionWorkerThread inspectionWorker; - ASSERT_TRUE( inspectionWorker.init( signalBufferPtr, outputCollectedData, 1000 ) ); - ASSERT_TRUE( inspectionWorker.start() ); + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread inspectionWorker( engine ); + initAndStartWorker( inspectionWorker ); // Create a set of DTCs and make sure they are NOT available in the collected data DTCInfo dtcInfo; dtcInfo.mDTCCodes.emplace_back( "P0143" ); dtcInfo.mDTCCodes.emplace_back( "C0196" ); dtcInfo.mSID = SID::STORED_DTC; - dtcInfo.receiveTime = fClock->systemTimeSinceEpochMs(); + dtcInfo.receiveTime = mClock->systemTimeSinceEpochMs(); ASSERT_TRUE( dtcInfo.hasItems() ); // Push the DTCs to the buffer - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( std::make_shared( dtcInfo ) ) ) ); // Prepare a condition to evaluate and expect the DTCs to be NOT collected. InspectionMatrixSignalCollectionInfo signal{}; signal.signalID = 1234; @@ -392,24 +420,28 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeH signal.minimumSampleIntervalMs = 0; signal.fixedWindowPeriod = 77777; signal.isConditionOnlySignal = false; + signal.signalType = SignalType::DOUBLE; collectionSchemes->conditions[0].signals.push_back( signal ); + // Condition contains signals + collectionSchemes->conditions[0].isStaticCondition = false; collectionSchemes->conditions[0].condition = getSignalsBiggerCondition( signal.signalID, 1 ).get(); // Make sure that DTCs should NOT be collected collectionSchemes->conditions[0].includeActiveDtcs = false; inspectionWorker.onChangeInspectionMatrix( consCollectionSchemes ); - Timestamp timestamp = fClock->systemTimeSinceEpochMs(); + Timestamp timestamp = mClock->systemTimeSinceEpochMs(); // Push the signals so that the condition is met CollectedSignalsGroup collectedSignalsGroup; - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1 ) ); - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.2 ) ); - collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 1.5 ) ); - ASSERT_TRUE( signalBufferPtr->push( CollectedDataFrame( collectedSignalsGroup ) ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.1, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 0.2, SignalType::DOUBLE ) ); + collectedSignalsGroup.push_back( CollectedSignal( signal.signalID, timestamp, 1.5, SignalType::DOUBLE ) ); + ASSERT_TRUE( signalBuffer->push( CollectedDataFrame( collectedSignalsGroup ) ) ); std::shared_ptr collectedData; + std::shared_ptr senderData; // Expect the data to be collected and has NO DTCs - WAIT_ASSERT_TRUE( outputCollectedData->pop( collectedData ) ); + WAIT_ASSERT_TRUE( popCollectedData( collectedData ) ); ASSERT_FALSE( collectedData->mDTCInfo.hasItems() ); inspectionWorker.stop(); @@ -417,7 +449,8 @@ TEST_F( CollectionInspectionWorkerThreadTest, ConsumeActiveDTCsCollectionSchemeH TEST_F( CollectionInspectionWorkerThreadTest, StartWithoutInit ) { - CollectionInspectionWorkerThread worker; + CollectionInspectionEngine engine; + CollectionInspectionWorkerThread worker( engine ); worker.onChangeInspectionMatrix( consCollectionSchemes ); } diff --git a/test/unit/CollectionSchemeManagerGtest.cpp b/test/unit/CollectionSchemeManagerGtest.cpp index d9023c73..7d480afb 100644 --- a/test/unit/CollectionSchemeManagerGtest.cpp +++ b/test/unit/CollectionSchemeManagerGtest.cpp @@ -2,14 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 #include "CANInterfaceIDTranslator.h" +#include "CheckinSender.h" #include "Clock.h" #include "ClockHandler.h" +#include "CollectionSchemeManager.h" #include "CollectionSchemeManagerMock.h" #include "CollectionSchemeManagerTest.h" // IWYU pragma: associated #include "Testing.h" +#include "TimeTypes.h" +#include #include #include #include +#include +#include namespace Aws { @@ -29,10 +35,10 @@ TEST( CollectionSchemeManagerGtest, RebuildUpdateAndTimeLineTest ) { // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strDecoderManifestIDCollectionScheme1 = "DM1"; - std::string strDecoderManifestIDCollectionScheme2 = "DM2"; + SyncID strDecoderManifestID1 = "DM1"; + SyncID strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; + SyncID strDecoderManifestIDCollectionScheme1 = "DM1"; + SyncID strDecoderManifestIDCollectionScheme2 = "DM2"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); Timestamp startIdleTime = currTime.systemTimeMs + 10; @@ -47,22 +53,16 @@ TEST( CollectionSchemeManagerGtest, RebuildUpdateAndTimeLineTest ) std::shared_ptr testPL = std::make_shared(); std::vector testCP, emptyCP; testCP.emplace_back( collectionScheme1 ); - NiceMock gmocktest( strDecoderManifestID1 ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), strDecoderManifestID1 ); EXPECT_CALL( *testPL, getCollectionSchemes() ) .WillOnce( ReturnRef( emptyCP ) ) .WillOnce( ReturnRef( testCP ) ) .WillOnce( ReturnRef( testCP ) ) - .WillOnce( ReturnRef( testCP ) ) .WillOnce( ReturnRef( testCP ) ); - EXPECT_CALL( *collectionScheme1, getDecoderManifestID() ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme2 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme2 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ); - EXPECT_CALL( *collectionScheme1, getStartTime() ) .WillOnce( Return( startIdleTime ) ) .WillOnce( Return( startEnableTime ) ) @@ -81,27 +81,17 @@ TEST( CollectionSchemeManagerGtest, RebuildUpdateAndTimeLineTest ) std::cout << COUT_GTEST_MGT << "1. Empty ICollectionCollectionScheme." << ANSI_TXT_DFT << std::endl; gmocktest.setCollectionSchemeList( testPL ); ASSERT_FALSE( gmocktest.rebuildMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 2. collectionScheme does not match currDMID - std::cout << COUT_GTEST_MGT << "2. CollectionScheme does not match current DM id." << ANSI_TXT_DFT << std::endl; + // 2. start time > currTime, add idle collectionScheme + std::cout << COUT_GTEST_MGT << " 3. start time > currTime, add idle collectionScheme" << ANSI_TXT_DFT << std::endl; ASSERT_FALSE( gmocktest.rebuildMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 3. DM matches, start time > currTime, add idle collectionScheme - std::cout << COUT_GTEST_MGT << " 3. DM matches, start time > currTime, add idle collectionScheme" << ANSI_TXT_DFT - << std::endl; - ASSERT_FALSE( gmocktest.rebuildMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 4. DM matches, start time <= currTime && stop time > currTime add enable collectionScheme - std::cout << COUT_GTEST_MGT - << "4. DM matches, start time <= currTime && stop time > currTime add enable collectionScheme" + // 3. start time <= currTime && stop time > currTime add enable collectionScheme + std::cout << COUT_GTEST_MGT << "4. start time <= currTime && stop time > currTime add enable collectionScheme" << ANSI_TXT_DFT << std::endl; ASSERT_TRUE( gmocktest.rebuildMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 5. DM matches, stop time < currTime already expired - std::cout << COUT_GTEST_MGT << "5. DM matches, stop time < currTime CollectionScheme already expired" - << ANSI_TXT_DFT << std::endl; + // 4. stop time < currTime already expired + std::cout << COUT_GTEST_MGT << "5. stop time < currTime CollectionScheme already expired" << ANSI_TXT_DFT + << std::endl; ASSERT_FALSE( gmocktest.rebuildMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); } /** @brief @@ -111,13 +101,13 @@ TEST( CollectionSchemeManagerGtest, RebuildUpdateAndTimeLineTest ) TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ADD_DELETE ) { // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; - std::string strCollectionSchemeIDCollectionScheme3 = "COLLECTIONSCHEME3"; + SyncID strDecoderManifestID1 = "DM1"; + SyncID strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; + SyncID strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; + SyncID strCollectionSchemeIDCollectionScheme3 = "COLLECTIONSCHEME3"; - std::string strDecoderManifestIDCollectionScheme1 = "DM1"; - std::string strDecoderManifestIDCollectionScheme2 = "DM2"; + SyncID strDecoderManifestIDCollectionScheme1 = "DM1"; + SyncID strDecoderManifestIDCollectionScheme2 = "DM2"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); Timestamp startIdleTime = currTime.systemTimeMs + 10; @@ -138,31 +128,17 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ADD_DELETE ) testCP.emplace_back( collectionScheme1 ); testCP.emplace_back( collectionScheme1 ); - NiceMock gmocktest( strDecoderManifestID1 ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), strDecoderManifestID1 ); EXPECT_CALL( *testPL, getCollectionSchemes() ) .WillOnce( ReturnRef( emptyCP ) ) - // addition DM does not match - .WillOnce( ReturnRef( testCP ) ) - // addition DM matches + // addition .WillOnce( ReturnRef( testCP ) ) // deletion .WillOnce( ReturnRef( emptyCP ) ); - EXPECT_CALL( *collectionScheme1, getDecoderManifestID() ) - // DM does not match - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme2 ) ) - // DM print for mismatch case - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme2 ) ) - // addition DM matches collectionScheme 1 - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - - // addition DM matches collectionScheme 2 - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - - // addition DM matches collectionScheme3 - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ); - EXPECT_CALL( *collectionScheme1, getStartTime() ) // addition add idle collectionScheme .WillOnce( Return( startIdleTime ) ) @@ -193,21 +169,13 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ADD_DELETE ) std::cout << COUT_GTEST_MGT << "1. Empty ICollectionScheme." << ANSI_TXT_DFT << std::endl; gmocktest.setCollectionSchemeList( testPL ); ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 2. collectionScheme does not match currDMID - std::cout << COUT_GTEST_MGT << "2. CollectionScheme does not match current DM id." << ANSI_TXT_DFT << std::endl; - ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 3. DM matches, add all collectionSchemes and drop the collectionScheme with bad setting - std::cout << COUT_GTEST_MGT - << " 3. DM matches, add all collectionSchemes and drop the collectionScheme with bad setting" + // 2. add all collectionSchemes and drop the collectionScheme with bad setting + std::cout << COUT_GTEST_MGT << " 3. add all collectionSchemes and drop the collectionScheme with bad setting" << ANSI_TXT_DFT << std::endl; ASSERT_TRUE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); - // 4. deletion + // 3. deletion std::cout << COUT_GTEST_MGT << "4. Delete all" << ANSI_TXT_DFT << std::endl; ASSERT_TRUE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); } /** @brief @@ -216,11 +184,6 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ADD_DELETE ) */ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_IDLE_BRANCHES ) { - // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - - std::string strDecoderManifestIDCollectionScheme1 = "DM1"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); Timestamp startIdleTime = currTime.systemTimeMs + 10; @@ -235,82 +198,31 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_IDLE_BRANCHES ) Timestamp start2Time = currTime.systemTimeMs - 50; Timestamp stop2Time = currTime.systemTimeMs - 10; - // build decodermanifest, collectionScheme, collectionSchemeList - std::shared_ptr testDM1 = std::make_shared(); - std::shared_ptr collectionScheme1 = std::make_shared(); - std::shared_ptr testPL = std::make_shared(); - std::vector testCP; - // 3 collectionSchemes on the list, same collectionScheme but different update - // idle->enabled, idle update and bad - testCP.emplace_back( collectionScheme1 ); - testCP.emplace_back( collectionScheme1 ); - testCP.emplace_back( collectionScheme1 ); - - // create a collectionScheme map - std::shared_ptr collectionSchemeIdle = std::make_shared(); - std::map map1 = { - { strCollectionSchemeIDCollectionScheme1, collectionSchemeIdle } }; - std::map mapEmpty; - // setup idleMap - NiceMock gmocktest( strDecoderManifestID1, mapEmpty, map1 ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); + auto idleCollectionScheme = + std::make_shared( "COLLECTIONSCHEME1", "DM1", startIdleTime, stopIdleTime ); - EXPECT_CALL( *testPL, getCollectionSchemes() ) - // update Idle collectionScheme DM matches - .WillOnce( ReturnRef( testCP ) ); + // test code + gmocktest.setCollectionSchemeList( + std::make_shared( std::vector{ idleCollectionScheme } ) ); + ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - EXPECT_CALL( *collectionScheme1, getDecoderManifestID() ) - // DM matches 3 collectionSchemes - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ); - - EXPECT_CALL( *collectionSchemeIdle, getStartTime() ) - // update bad setting - .WillRepeatedly( Return( startIdleTime ) ); - EXPECT_CALL( *collectionSchemeIdle, getExpiryTime() ) - // update bad setting - .WillRepeatedly( Return( stopIdleTime ) ); - using ::testing::InSequence; - { - InSequence s1; - // 1st update bad setting - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( start2Time ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stop2Time ) ); - - // 2nd update new idle time - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( start1Time ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stop1Time ) ); - - // 3rd update move to enable - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( startEnableTime ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stopEnableTime ) ); - } + // 1st update bad setting + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", start2Time, stop2Time ) } ) ); + ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - EXPECT_CALL( *collectionScheme1, getCollectionSchemeID() ) - // addition - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ) - // printExistingCollectionSchemes - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ); + // 2nd update new idle time + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", start1Time, stop1Time ) } ) ); + ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - // test code - gmocktest.setCollectionSchemeList( testPL ); - gmocktest.sendCheckin(); + // 3rd update move to enable + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", startEnableTime, stopEnableTime ) } ) ); ASSERT_TRUE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); } /** @brief @@ -333,11 +245,6 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ENABLED_BRANCHES ) * */ - // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - - std::string strDecoderManifestIDCollectionScheme1 = "DM1"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); Timestamp startExpTime = currTime.systemTimeMs + 10; @@ -352,125 +259,31 @@ TEST( CollectionSchemeManagerGtest, updateMapsandTimeLineTest_ENABLED_BRANCHES ) Timestamp start2Time = currTime.systemTimeMs - 50; Timestamp stop2Time = currTime.systemTimeMs + 100; - // build decodermanifest, collectionScheme, collectionSchemeList - std::shared_ptr testDM1 = std::make_shared(); - std::shared_ptr collectionScheme1 = std::make_shared(); - std::shared_ptr testPL = std::make_shared(); - std::vector testCP; - // 3 collectionSchemes on the list, same collectionScheme but different update - // idle->enabled, idle update and bad - testCP.emplace_back( collectionScheme1 ); - testCP.emplace_back( collectionScheme1 ); - testCP.emplace_back( collectionScheme1 ); - - // create a collectionScheme map - std::shared_ptr collectionSchemeEnabled = std::make_shared(); - std::map map1 = { - { strCollectionSchemeIDCollectionScheme1, collectionSchemeEnabled } }; - std::map mapEmpty; - // setup idleMap - NiceMock gmocktest( strDecoderManifestID1, map1, mapEmpty ); - - EXPECT_CALL( *testPL, getCollectionSchemes() ) - // update Idle collectionScheme DM matches - .WillOnce( ReturnRef( testCP ) ); - - EXPECT_CALL( *collectionScheme1, getDecoderManifestID() ) - // DM matches 3 collectionSchemes - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strDecoderManifestIDCollectionScheme1 ) ); - - EXPECT_CALL( *collectionSchemeEnabled, getStartTime() ) - // update bad setting - .WillRepeatedly( Return( startEnableTime ) ); - EXPECT_CALL( *collectionSchemeEnabled, getExpiryTime() ) - // update bad setting - .WillRepeatedly( Return( stopEnableTime ) ); - using ::testing::InSequence; - { - InSequence s1; - // 1st update bad setting - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( start1Time ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stop1Time ) ); - - // 2nd update new idle time - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( start2Time ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stop2Time ) ); - - // 3rd update move to enable - EXPECT_CALL( *collectionScheme1, getStartTime() ) - // update bad setting - .WillOnce( Return( startExpTime ) ); - EXPECT_CALL( *collectionScheme1, getExpiryTime() ) - // update bad setting - .WillOnce( Return( stopExpTime ) ); - } - - EXPECT_CALL( *collectionScheme1, getCollectionSchemeID() ) - // addition - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ) - .WillOnce( ReturnRef( strCollectionSchemeIDCollectionScheme1 ) ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), "COLLECTIONSCHEME1" ); + auto enabledCollectionScheme = + std::make_shared( "COLLECTIONSCHEME1", "DM1", startEnableTime, stopEnableTime ); // test code - gmocktest.setCollectionSchemeList( testPL ); - gmocktest.sendCheckin(); + gmocktest.setCollectionSchemeList( + std::make_shared( std::vector{ enabledCollectionScheme } ) ); ASSERT_TRUE( gmocktest.updateMapsandTimeLine( currTime ) ); - gmocktest.sendCheckin(); -} - -/** @brief - * This test validates the checkin interval when no active collection scheme is in the system - */ -TEST( CollectionSchemeManagerGtest, checkTimeLineTest_CHECKIN_UNFOUNDCOLLECTIONSCHEME ) -{ - // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; - - std::shared_ptr testClock = ClockHandler::getClock(); - TimePoint currTime = testClock->timeSinceEpoch(); - // create empty collectionScheme map - std::map mapEmpty; - std::shared_ptr collectionScheme2 = std::make_shared(); - std::map mapIdle = { - { strCollectionSchemeIDCollectionScheme2, collectionScheme2 } }; - - // setup maps - NiceMock gmocktest( strDecoderManifestID1, mapEmpty, mapIdle ); - EXPECT_CALL( *collectionScheme2, getStartTime() ).WillOnce( Return( currTime.systemTimeMs + 20 ) ); + // 2nd update new idle time + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", start1Time, stop1Time ) } ) ); + ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - // create mTimeLine - std::priority_queue, std::greater> testTimeLine; - TimeData dataPair = { currTime, "Checkin" }; - testTimeLine.push( dataPair ); - dataPair = { currTime, strCollectionSchemeIDCollectionScheme1 }; - testTimeLine.push( dataPair ); - dataPair = { currTime, strCollectionSchemeIDCollectionScheme2 }; - testTimeLine.push( dataPair ); + // 1st update bad setting + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", start2Time, stop2Time ) } ) ); + ASSERT_FALSE( gmocktest.updateMapsandTimeLine( currTime ) ); - // test code - CANInterfaceIDTranslator canIDTranslator; - gmocktest.init( 200, nullptr, canIDTranslator ); - gmocktest.setTimeLine( testTimeLine ); - gmocktest.sendCheckin(); - // first case collectionScheme1 not found - ASSERT_FALSE( gmocktest.checkTimeLine( currTime ) ); - // second case collectionScheme2 does not have time matched - ASSERT_FALSE( gmocktest.checkTimeLine( currTime + 200 ) ); - // branch out startTime > currTime - ASSERT_FALSE( gmocktest.checkTimeLine( currTime + 200 ) ); + // 3rd update move to enable + gmocktest.setCollectionSchemeList( std::make_shared( std::vector{ + std::make_shared( "COLLECTIONSCHEME1", "DM1", startExpTime, stopExpTime ) } ) ); + ASSERT_TRUE( gmocktest.updateMapsandTimeLine( currTime ) ); } /** @brief @@ -479,9 +292,9 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_CHECKIN_UNFOUNDCOLLECTIONS TEST( CollectionSchemeManagerGtest, checkTimeLineTest_IDLE_BRANCHES ) { // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; + SyncID strDecoderManifestID1 = "DM1"; + SyncID strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; + SyncID strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); @@ -489,13 +302,18 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_IDLE_BRANCHES ) // create empty collectionScheme map std::shared_ptr collectionScheme1 = std::make_shared(); std::shared_ptr collectionScheme2 = std::make_shared(); - std::map mapEmpty; - std::map mapIdle = { - { strCollectionSchemeIDCollectionScheme1, collectionScheme1 }, - { strCollectionSchemeIDCollectionScheme2, collectionScheme2 } }; + std::map mapEmpty; + std::map mapIdle = { { strCollectionSchemeIDCollectionScheme1, collectionScheme1 }, + { strCollectionSchemeIDCollectionScheme2, collectionScheme2 } }; // setup maps - NiceMock gmocktest( strDecoderManifestID1, mapEmpty, mapIdle ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( nullptr, + canIDTranslator, + std::make_shared( nullptr ), + strDecoderManifestID1, + mapEmpty, + mapIdle ); EXPECT_CALL( *collectionScheme1, getStartTime() ) .WillOnce( Return( currTime.systemTimeMs ) ) .WillOnce( Return( currTime.systemTimeMs ) ); @@ -520,10 +338,7 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_IDLE_BRANCHES ) testTimeLine.push( dataPair ); // test code - CANInterfaceIDTranslator canIDTranslator; - gmocktest.init( 200, nullptr, canIDTranslator ); gmocktest.setTimeLine( testTimeLine ); - gmocktest.sendCheckin(); ASSERT_TRUE( gmocktest.checkTimeLine( currTime ) ); ASSERT_TRUE( gmocktest.checkTimeLine( currTime + 20 ) ); // branch out startTime > currTime @@ -536,9 +351,9 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_IDLE_BRANCHES ) TEST( CollectionSchemeManagerGtest, checkTimeLineTest_ENABLED_BRANCHES ) { // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; + SyncID strDecoderManifestID1 = "DM1"; + SyncID strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; + SyncID strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; std::shared_ptr testClock = ClockHandler::getClock(); TimePoint currTime = testClock->timeSinceEpoch(); @@ -546,13 +361,19 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_ENABLED_BRANCHES ) // create empty collectionScheme map std::shared_ptr collectionScheme1 = std::make_shared(); std::shared_ptr collectionScheme2 = std::make_shared(); - std::map mapEmpty; - std::map mapEnable = { + std::map mapEmpty; + std::map mapEnable = { { strCollectionSchemeIDCollectionScheme1, collectionScheme1 }, { strCollectionSchemeIDCollectionScheme2, collectionScheme2 } }; // setup maps - NiceMock gmocktest( strDecoderManifestID1, mapEnable, mapEmpty ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( nullptr, + canIDTranslator, + std::make_shared( nullptr ), + strDecoderManifestID1, + mapEnable, + mapEmpty ); EXPECT_CALL( *collectionScheme1, getExpiryTime() ) .WillOnce( Return( currTime.systemTimeMs ) ) .WillOnce( Return( currTime.systemTimeMs ) ); @@ -577,10 +398,7 @@ TEST( CollectionSchemeManagerGtest, checkTimeLineTest_ENABLED_BRANCHES ) testTimeLine.push( dataPair ); // test code - CANInterfaceIDTranslator canIDTranslator; - gmocktest.init( 200, nullptr, canIDTranslator ); gmocktest.setTimeLine( testTimeLine ); - gmocktest.sendCheckin(); ASSERT_TRUE( gmocktest.checkTimeLine( currTime ) ); ASSERT_TRUE( gmocktest.checkTimeLine( currTime + 20 ) ); // branch out startTime > currTime diff --git a/test/unit/CollectionSchemeManagerTest.cpp b/test/unit/CollectionSchemeManagerTest.cpp index 0d0ca8cb..5eb79180 100644 --- a/test/unit/CollectionSchemeManagerTest.cpp +++ b/test/unit/CollectionSchemeManagerTest.cpp @@ -3,12 +3,23 @@ #include "CollectionSchemeManagerTest.h" #include "CANInterfaceIDTranslator.h" +#include "CacheAndPersist.h" +#include "CheckinSender.h" #include "Clock.h" #include "ClockHandler.h" +#include "CollectionSchemeManagerMock.h" +#include "SchemaListener.h" +#include "Testing.h" +#include "TimeTypes.h" #include "WaitUntil.h" +#include "collection_schemes.pb.h" +#include "decoder_manifest.pb.h" #include +#include +#include #include #include +#include #include namespace Aws @@ -16,26 +27,104 @@ namespace Aws namespace IoTFleetWise { -/**********************test body ***********************************************/ -TEST( CollectionSchemeManagerTest, StopMainTest ) +using ::testing::_; +using ::testing::InvokeArgument; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::StrictMock; + +class SchemaListenerMock : public SchemaListener +{ +public: + void + sendCheckin( const std::vector &documentARNs, OnCheckinSentCallback callback ) override + { + mockedSendCheckin( documentARNs, callback ); + std::lock_guard lock( sentDocumentsMutex ); + mSentDocuments.push_back( documentARNs ); + } + + MOCK_METHOD( void, mockedSendCheckin, ( const std::vector &documentARNs, OnCheckinSentCallback callback ) ); + + std::vector> + getSentDocuments() + { + std::lock_guard lock( sentDocumentsMutex ); + return mSentDocuments; + } + + int + getLastSentDocuments( std::vector &sentDocuments ) + { + std::lock_guard lock( sentDocumentsMutex ); + if ( mSentDocuments.empty() ) + { + return -1; + } + sentDocuments = mSentDocuments.back(); + return static_cast( sentDocuments.size() ); + } + +private: + std::vector> mSentDocuments; + std::mutex sentDocumentsMutex; +}; + +class CollectionSchemeManagerTest : public ::testing::Test +{ +protected: + CollectionSchemeManagerTest() + : mCollectionSchemeManager( nullptr, mCanIDTranslator, std::make_shared( nullptr ) ) + { + } + + void + SetUp() override + { + mCollectionSchemeManager.subscribeToInspectionMatrixChange( + [&]( const std::shared_ptr &inspectionMatrix ) { + mReceivedInspectionMatrices.emplace_back( inspectionMatrix ); + } ); + } + + void + TearDown() override + { + WAIT_ASSERT_TRUE( mCollectionSchemeManager.disconnect() ); + } + + CANInterfaceIDTranslator mCanIDTranslator; + CollectionSchemeManagerWrapper mCollectionSchemeManager; + std::vector> mReceivedInspectionMatrices; + std::shared_ptr mTestClock = ClockHandler::getClock(); +}; + +std::vector +getSignalIdsFromCondition( const ConditionWithCollectedData &condition ) +{ + std::vector signalIds; + for ( const auto &signal : condition.signals ) + { + signalIds.push_back( signal.signalID ); + } + return signalIds; +} + +TEST_F( CollectionSchemeManagerTest, StopMain ) { - CANInterfaceIDTranslator canIDTranslator; - CollectionSchemeManagerTest test; - test.init( 50, nullptr, canIDTranslator ); - ASSERT_TRUE( test.connect() ); + ASSERT_TRUE( mCollectionSchemeManager.connect() ); /* stopping idling main thread */ - WAIT_ASSERT_TRUE( test.disconnect() ); + WAIT_ASSERT_TRUE( mCollectionSchemeManager.disconnect() ); /* build DMs */ - ASSERT_TRUE( test.connect() ); + ASSERT_TRUE( mCollectionSchemeManager.connect() ); IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); std::vector testList1; /* build collectionScheme list1 */ - std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ - TimePoint currTime = testClock->timeSinceEpoch(); + TimePoint currTime = mTestClock->timeSinceEpoch(); Timestamp startTime = currTime.systemTimeMs + SECOND_TO_MILLISECOND( 1 ); Timestamp stopTime = startTime + SECOND_TO_MILLISECOND( 25 ); ICollectionSchemePtr collectionScheme = @@ -44,65 +133,60 @@ TEST( CollectionSchemeManagerTest, StopMainTest ) std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); /* create ICollectionSchemeList */ - test.mPlTest = std::make_shared( testList1 ); + mCollectionSchemeManager.mPlTest = std::make_shared( testList1 ); /* sending lists and dm to PM */ - test.mDmTest = testDM1; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM1; + mCollectionSchemeManager.myInvokeDecoderManifest(); + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + mCollectionSchemeManager.myInvokeCollectionScheme(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); - test.myInvokeCollectionScheme(); /* stopping main thread servicing a collectionScheme ending in 25 seconds */ - WAIT_ASSERT_TRUE( test.disconnect() ); + WAIT_ASSERT_TRUE( mCollectionSchemeManager.disconnect() ); } -TEST( CollectionSchemeManagerTest, CollectionSchemeUpdateCallBackTest ) +TEST_F( CollectionSchemeManagerTest, CollectionSchemeUpdateCallBack ) { - CollectionSchemeManagerTest test; std::vector emptyList; - CANInterfaceIDTranslator canIDTranslator; - test.init( 50, nullptr, canIDTranslator ); - test.setmCollectionSchemeAvailable( false ); - test.setmProcessCollectionScheme( false ); + mCollectionSchemeManager.setmCollectionSchemeAvailable( false ); + mCollectionSchemeManager.setmProcessCollectionScheme( false ); // pl is null - test.mPlTest = nullptr; - test.myInvokeCollectionScheme(); - ASSERT_TRUE( test.getmCollectionSchemeAvailable() ); - test.updateAvailable(); - ASSERT_FALSE( test.getmCollectionSchemeAvailable() ); - ASSERT_FALSE( test.getmProcessCollectionScheme() ); + mCollectionSchemeManager.mPlTest = nullptr; + mCollectionSchemeManager.myInvokeCollectionScheme(); + ASSERT_TRUE( mCollectionSchemeManager.getmCollectionSchemeAvailable() ); + mCollectionSchemeManager.updateAvailable(); + ASSERT_FALSE( mCollectionSchemeManager.getmCollectionSchemeAvailable() ); + ASSERT_FALSE( mCollectionSchemeManager.getmProcessCollectionScheme() ); // pl is valid - test.mPlTest = std::make_shared( emptyList ); - test.myInvokeCollectionScheme(); - ASSERT_TRUE( test.getmCollectionSchemeAvailable() ); - test.updateAvailable(); - ASSERT_FALSE( test.getmCollectionSchemeAvailable() ); - ASSERT_TRUE( test.getmProcessCollectionScheme() ); + mCollectionSchemeManager.mPlTest = std::make_shared( emptyList ); + mCollectionSchemeManager.myInvokeCollectionScheme(); + ASSERT_TRUE( mCollectionSchemeManager.getmCollectionSchemeAvailable() ); + mCollectionSchemeManager.updateAvailable(); + ASSERT_FALSE( mCollectionSchemeManager.getmCollectionSchemeAvailable() ); + ASSERT_TRUE( mCollectionSchemeManager.getmProcessCollectionScheme() ); } -TEST( CollectionSchemeManagerTest, DecoderManifestUpdateCallBackTest ) +TEST_F( CollectionSchemeManagerTest, DecoderManifestUpdateCallBack ) { - CollectionSchemeManagerTest test; - CANInterfaceIDTranslator canIDTranslator; - test.init( 50, nullptr, canIDTranslator ); - test.setmDecoderManifestAvailable( false ); - test.setmProcessDecoderManifest( false ); + mCollectionSchemeManager.setmDecoderManifestAvailable( false ); + mCollectionSchemeManager.setmProcessDecoderManifest( false ); // dm is null - test.mDmTest = nullptr; - test.myInvokeDecoderManifest(); - ASSERT_TRUE( test.getmDecoderManifestAvailable() ); - test.updateAvailable(); - ASSERT_FALSE( test.getmDecoderManifestAvailable() ); - ASSERT_FALSE( test.getmProcessDecoderManifest() ); + mCollectionSchemeManager.mDmTest = nullptr; + mCollectionSchemeManager.myInvokeDecoderManifest(); + ASSERT_TRUE( mCollectionSchemeManager.getmDecoderManifestAvailable() ); + mCollectionSchemeManager.updateAvailable(); + ASSERT_FALSE( mCollectionSchemeManager.getmDecoderManifestAvailable() ); + ASSERT_FALSE( mCollectionSchemeManager.getmProcessDecoderManifest() ); // pl is valid - test.mDmTest = std::make_shared( "" ); - test.myInvokeDecoderManifest(); - ASSERT_TRUE( test.getmDecoderManifestAvailable() ); - test.updateAvailable(); - ASSERT_FALSE( test.getmDecoderManifestAvailable() ); - ASSERT_TRUE( test.getmProcessDecoderManifest() ); + mCollectionSchemeManager.mDmTest = std::make_shared( "" ); + mCollectionSchemeManager.myInvokeDecoderManifest(); + ASSERT_TRUE( mCollectionSchemeManager.getmDecoderManifestAvailable() ); + mCollectionSchemeManager.updateAvailable(); + ASSERT_FALSE( mCollectionSchemeManager.getmDecoderManifestAvailable() ); + ASSERT_TRUE( mCollectionSchemeManager.getmProcessDecoderManifest() ); } -TEST( CollectionSchemeManagerTest, MockProducerTest ) +TEST_F( CollectionSchemeManagerTest, MockProducer ) { /* * This is the integration test of PM. A mock producer is creating mocking CollectionScheme Ingestion sending @@ -132,10 +216,7 @@ TEST( CollectionSchemeManagerTest, MockProducerTest ) * watching mTimeLine run to complete all collectionSchemes. */ /* start PM main thread */ - CollectionSchemeManagerTest test; - CANInterfaceIDTranslator canIDTranslator; - test.init( 50, nullptr, canIDTranslator ); - ASSERT_TRUE( test.connect() ); + ASSERT_TRUE( mCollectionSchemeManager.connect() ); /* build DMs */ IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); @@ -143,8 +224,7 @@ TEST( CollectionSchemeManagerTest, MockProducerTest ) std::vector testList1, testList2, testList3; /* build collectionScheme list1 */ - std::shared_ptr testClock = ClockHandler::getClock(); - TimePoint currTime = testClock->timeSinceEpoch(); + TimePoint currTime = mTestClock->timeSinceEpoch(); Timestamp startTime = currTime.systemTimeMs + 100; Timestamp stopTime = startTime + 500; ICollectionSchemePtr collectionScheme = @@ -156,53 +236,53 @@ TEST( CollectionSchemeManagerTest, MockProducerTest ) collectionScheme = std::make_shared( "COLLECTIONSCHEME2", "DM1", startTime, stopTime ); testList1.emplace_back( collectionScheme ); /* create ICollectionSchemeList */ - test.mPlTest = std::make_shared( testList1 ); + mCollectionSchemeManager.mPlTest = std::make_shared( testList1 ); //////////////////// test starts here//////////////////////////////// /* sending lists and dm to PM */ std::cout << COUT_GTEST_MGT << "Step1: send DM1 and COLLECTIONSCHEME1 and COLLECTIONSCHEME2 " << ANSI_TXT_DFT << std::endl; - test.mDmTest = testDM1; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM1; + mCollectionSchemeManager.myInvokeDecoderManifest(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.myInvokeCollectionScheme(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); // remove CollectionScheme1, don't send DM std::cout << COUT_GTEST_MGT << "Step2: remove COLLECTIONSCHEME1 " << ANSI_TXT_DFT << std::endl; testList2.emplace_back( testList1[1] ); - test.mPlTest = std::make_shared( testList2 ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.mPlTest = std::make_shared( testList2 ); + mCollectionSchemeManager.myInvokeCollectionScheme(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); /* add COLLECTIONSCHEME3 */ std::cout << COUT_GTEST_MGT << "Step3: add COLLECTIONSCHEME3 " << ANSI_TXT_DFT << std::endl; - currTime = testClock->timeSinceEpoch(); + currTime = mTestClock->timeSinceEpoch(); startTime = currTime.systemTimeMs + 100; stopTime = startTime + 500; collectionScheme = std::make_shared( "COLLECTIONSCHEME3", "DM1", startTime, stopTime ); testList2.emplace_back( collectionScheme ); - test.mPlTest = std::make_shared( testList2 ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.mPlTest = std::make_shared( testList2 ); + mCollectionSchemeManager.myInvokeCollectionScheme(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); /* send DM2 and list2(of DM1) to PM, PM will stop all collectionSchemes */ std::cout << COUT_GTEST_MGT << "Step4: send DM2 " << ANSI_TXT_DFT << std::endl; - test.mDmTest = testDM2; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM2; + mCollectionSchemeManager.myInvokeDecoderManifest(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); std::cout << COUT_GTEST_MGT << "Step5: send DM1 again " << ANSI_TXT_DFT << std::endl; - test.mDmTest = testDM1; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM1; + mCollectionSchemeManager.myInvokeDecoderManifest(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); std::cout << COUT_GTEST_MGT << "Step6: send DM2 again " << ANSI_TXT_DFT << std::endl; - test.mDmTest = testDM2; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM2; + mCollectionSchemeManager.myInvokeDecoderManifest(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); /* build list3 with DM2 */ - currTime = testClock->timeSinceEpoch(); + currTime = mTestClock->timeSinceEpoch(); startTime = currTime.systemTimeMs + 100; stopTime = startTime + 500; collectionScheme = std::make_shared( "COLLECTIONSCHEME4", "DM2", startTime, stopTime ); @@ -213,35 +293,31 @@ TEST( CollectionSchemeManagerTest, MockProducerTest ) testList3.emplace_back( collectionScheme ); /* send list3 to PM, PM will start rebuilding all collectionSchemes */ std::cout << COUT_GTEST_MGT << "Step7: send COLLECTIONSCHEME4 and COLLECTIONSCHEME5 " << ANSI_TXT_DFT << std::endl; - test.mPlTest = std::make_shared( testList3 ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.mPlTest = std::make_shared( testList3 ); + mCollectionSchemeManager.myInvokeCollectionScheme(); std::this_thread::sleep_for( std::chrono::milliseconds( 200 ) ); /* send empty list of any DM */ std::cout << COUT_GTEST_MGT << "Step7: send empty collectionSchemeList " << ANSI_TXT_DFT << std::endl; testList2.clear(); - test.mPlTest = std::make_shared( testList2 ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.mPlTest = std::make_shared( testList2 ); + mCollectionSchemeManager.myInvokeCollectionScheme(); - WAIT_ASSERT_TRUE( test.disconnect() ); + WAIT_ASSERT_TRUE( mCollectionSchemeManager.disconnect() ); } -TEST( CollectionSchemeManagerTest, getCollectionSchemeArns ) +TEST_F( CollectionSchemeManagerTest, getCollectionSchemeArns ) { - CANInterfaceIDTranslator canIDTranslator; - CollectionSchemeManagerTest test; - test.init( 50, nullptr, canIDTranslator ); - ASSERT_TRUE( test.connect() ); + ASSERT_TRUE( mCollectionSchemeManager.connect() ); - ASSERT_EQ( test.getCollectionSchemeArns(), std::vector() ); + ASSERT_EQ( mCollectionSchemeManager.getCollectionSchemeArns(), std::vector() ); /* build DMs */ IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); std::vector testList1; /* build collectionScheme list1 */ - std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ - TimePoint currTime = testClock->timeSinceEpoch(); + TimePoint currTime = mTestClock->timeSinceEpoch(); Timestamp startTime = currTime.systemTimeMs + SECOND_TO_MILLISECOND( 1 ); Timestamp stopTime = startTime + SECOND_TO_MILLISECOND( 25 ); ICollectionSchemePtr collectionScheme = @@ -250,18 +326,290 @@ TEST( CollectionSchemeManagerTest, getCollectionSchemeArns ) std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); /* create ICollectionSchemeList */ - test.mPlTest = std::make_shared( testList1 ); + mCollectionSchemeManager.mPlTest = std::make_shared( testList1 ); /* sending lists and dm to PM */ - test.mDmTest = testDM1; - test.myInvokeDecoderManifest(); + mCollectionSchemeManager.mDmTest = testDM1; + mCollectionSchemeManager.myInvokeDecoderManifest(); std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); - test.myInvokeCollectionScheme(); + mCollectionSchemeManager.myInvokeCollectionScheme(); - WAIT_ASSERT_EQ( test.getCollectionSchemeArns(), std::vector( { "COLLECTIONSCHEME1" } ) ); + WAIT_ASSERT_EQ( mCollectionSchemeManager.getCollectionSchemeArns(), + std::vector( { "COLLECTIONSCHEME1" } ) ); /* stopping main thread servicing a collectionScheme ending in 25 seconds */ - WAIT_ASSERT_TRUE( test.disconnect() ); + WAIT_ASSERT_TRUE( mCollectionSchemeManager.disconnect() ); +} + +TEST_F( CollectionSchemeManagerTest, SendCheckinPeriodically ) +{ + uint32_t checkinIntervalMs = 100; + auto schemaListenerMock = std::make_shared>(); + auto checkinSender = std::make_shared( schemaListenerMock, checkinIntervalMs ); + CollectionSchemeManagerWrapper collectionSchemeManager( nullptr, mCanIDTranslator, checkinSender ); + + EXPECT_CALL( *schemaListenerMock, mockedSendCheckin( _, _ ) ).WillRepeatedly( InvokeArgument<1>( true ) ); + + std::vector sentDocuments; + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + ASSERT_TRUE( checkinSender->start() ); + + // No checkin should be sent until CollectionSchemeManager is started, because if CollectionSchemeManager + // restores data from persistency layer, the first checkin message should contain the restored + // documents instead of being an empty checkin. + std::this_thread::sleep_for( std::chrono::milliseconds( checkinIntervalMs * 2 ) ); + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + + ASSERT_TRUE( collectionSchemeManager.connect() ); + + // Now that CollectionSchemeManager is started and nothing is persisted, the first checkin should + // be empty + WAIT_ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), 0 ); + + /* build DMs */ + IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); + std::vector testList1; + /* build collectionScheme list1 */ + TimePoint currTime = mTestClock->timeSinceEpoch(); + Timestamp startTime = currTime.systemTimeMs + SECOND_TO_MILLISECOND( 1 ); + Timestamp stopTime = startTime + SECOND_TO_MILLISECOND( 25 ); + testList1.emplace_back( + std::make_shared( "COLLECTIONSCHEME1", "DM1", startTime, stopTime ) ); + testList1.emplace_back( + std::make_shared( "COLLECTIONSCHEME2", "DM1", startTime, stopTime ) ); + + collectionSchemeManager.mPlTest = std::make_shared( testList1 ); + collectionSchemeManager.mDmTest = testDM1; + collectionSchemeManager.myInvokeDecoderManifest(); + collectionSchemeManager.myInvokeCollectionScheme(); + + WAIT_ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), 3 ); + ASSERT_EQ( sentDocuments[0], "COLLECTIONSCHEME1" ); + ASSERT_EQ( sentDocuments[1], "COLLECTIONSCHEME2" ); + ASSERT_EQ( sentDocuments[2], "DM1" ); + + auto numOfMessagesSent = schemaListenerMock->getSentDocuments().size(); + + std::this_thread::sleep_for( std::chrono::milliseconds( checkinIntervalMs * 5 ) ); + + // Make sure checkin is sent periodically, but leave some margin for small timing differences + ASSERT_GE( schemaListenerMock->getSentDocuments().size(), numOfMessagesSent + 4U ); + ASSERT_LT( schemaListenerMock->getSentDocuments().size(), numOfMessagesSent + 6U ); +} + +TEST_F( CollectionSchemeManagerTest, SendFirstCheckinWithPersistedDocuments ) +{ + auto storage = createCacheAndPersist(); + + // Store some documents + { + + Schemas::DecoderManifestMsg::DecoderManifest protoDM; + protoDM.set_sync_id( "DM1" ); + Schemas::DecoderManifestMsg::CANSignal *protoCANSignalA = protoDM.add_can_signals(); + protoCANSignalA->set_signal_id( 3908 ); + + std::string protoSerializedBuffer; + ASSERT_TRUE( protoDM.SerializeToString( &protoSerializedBuffer ) ); + ASSERT_EQ( storage->write( reinterpret_cast( protoSerializedBuffer.data() ), + protoSerializedBuffer.size(), + DataType::DECODER_MANIFEST ), + ErrorCode::SUCCESS ); + } + + { + Schemas::CollectionSchemesMsg::CollectionSchemes protoCollectionSchemesMsg; + auto collectionScheme1 = protoCollectionSchemesMsg.add_collection_schemes(); + collectionScheme1->set_campaign_sync_id( "COLLECTIONSCHEME1" ); + collectionScheme1->set_decoder_manifest_sync_id( "DM1" ); + Schemas::CollectionSchemesMsg::TimeBasedCollectionScheme *timeBasedCollectionScheme = + collectionScheme1->mutable_time_based_collection_scheme(); + timeBasedCollectionScheme->set_time_based_collection_scheme_period_ms( 5000 ); + collectionScheme1->set_expiry_time_ms_epoch( mTestClock->systemTimeSinceEpochMs() + 30000 ); + auto collectionScheme2 = protoCollectionSchemesMsg.add_collection_schemes(); + collectionScheme2->set_campaign_sync_id( "COLLECTIONSCHEME2" ); + collectionScheme2->set_decoder_manifest_sync_id( "DM1" ); + collectionScheme2->set_expiry_time_ms_epoch( mTestClock->systemTimeSinceEpochMs() + 30000 ); + timeBasedCollectionScheme = collectionScheme2->mutable_time_based_collection_scheme(); + timeBasedCollectionScheme->set_time_based_collection_scheme_period_ms( 5000 ); + + std::string protoSerializedBuffer; + ASSERT_TRUE( protoCollectionSchemesMsg.SerializeToString( &protoSerializedBuffer ) ); + ASSERT_EQ( storage->write( reinterpret_cast( protoSerializedBuffer.data() ), + protoSerializedBuffer.size(), + DataType::COLLECTION_SCHEME_LIST ), + ErrorCode::SUCCESS ); + } + + uint32_t checkinIntervalMs = 100; + auto schemaListenerMock = std::make_shared>(); + auto checkinSender = std::make_shared( schemaListenerMock, checkinIntervalMs ); + CollectionSchemeManagerWrapper collectionSchemeManager( storage, mCanIDTranslator, checkinSender ); + + EXPECT_CALL( *schemaListenerMock, mockedSendCheckin( _, _ ) ).WillRepeatedly( InvokeArgument<1>( true ) ); + + std::vector sentDocuments; + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + ASSERT_TRUE( checkinSender->start() ); + + // No checkin should be sent until CollectionSchemeManager is started, because if CollectionSchemeManager + // restores data from persistency layer, the first checkin message should contain the restored + // documents instead of being an empty checkin. + std::this_thread::sleep_for( std::chrono::milliseconds( checkinIntervalMs * 2 ) ); + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + + ASSERT_TRUE( collectionSchemeManager.connect() ); + + // Now that CollectionSchemeManager is started and restored persisted data, the first checkin should + // contain the restored documents. + WAIT_ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), 3 ); + ASSERT_EQ( sentDocuments[0], "COLLECTIONSCHEME1" ); + ASSERT_EQ( sentDocuments[1], "COLLECTIONSCHEME2" ); + ASSERT_EQ( sentDocuments[2], "DM1" ); +} + +TEST_F( CollectionSchemeManagerTest, RetryCheckinOnFailure ) +{ + uint32_t checkinIntervalMs = 100; + auto schemaListenerMock = std::make_shared>(); + auto checkinSender = std::make_shared( schemaListenerMock, checkinIntervalMs ); + CollectionSchemeManagerWrapper collectionSchemeManager( nullptr, mCanIDTranslator, checkinSender ); + + EXPECT_CALL( *schemaListenerMock, mockedSendCheckin( _, _ ) ).WillRepeatedly( InvokeArgument<1>( true ) ); + + std::vector sentDocuments; + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + ASSERT_TRUE( checkinSender->start() ); + + // No checkin should be sent until CollectionSchemeManager is started, because if CollectionSchemeManager + // restores data from persistency layer, the first checkin message should contain the restored + // documents instead of being an empty checkin. + std::this_thread::sleep_for( std::chrono::milliseconds( checkinIntervalMs * 2 ) ); + ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), -1 ); + + EXPECT_CALL( *schemaListenerMock, mockedSendCheckin( _, _ ) ).WillRepeatedly( InvokeArgument<1>( false ) ); + + ASSERT_TRUE( collectionSchemeManager.connect() ); + + WAIT_ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), 0 ); + + /* build DMs */ + IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); + collectionSchemeManager.mDmTest = testDM1; + collectionSchemeManager.myInvokeDecoderManifest(); + + WAIT_ASSERT_EQ( schemaListenerMock->getLastSentDocuments( sentDocuments ), 1 ); + ASSERT_EQ( sentDocuments[0], "DM1" ); +} + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA +TEST_F( CollectionSchemeManagerTest, ReceiveCollectionSchemeUpdateWithExistingSchemes ) +{ + IDecoderManifestPtr testDM1 = std::make_shared( "DM1" ); + mCollectionSchemeManager.mDmTest = testDM1; + mCollectionSchemeManager.myInvokeDecoderManifest(); + ASSERT_TRUE( mCollectionSchemeManager.connect() ); + + WAIT_ASSERT_EQ( mReceivedInspectionMatrices.size(), 1U ); + + auto currentTimeMs = mTestClock->timeSinceEpoch().systemTimeMs; + auto startTimeInNearFuture = currentTimeMs + 500; + auto stopTimeInDistantFuture = currentTimeMs + 200000; + auto startTimeInThePast = currentTimeMs - 1000; + std::vector signals1 = { SignalCollectionInfo{ 0xFFFF0000 }, + SignalCollectionInfo{ 0xFFFF0001 } }; + ICollectionScheme::PartialSignalIDLookup partialSignalIDLookup1{ + { 0xFFFF0000, std::pair( 0x2000000, SignalPath{ 1, 2, 5 } ) }, + { 0xFFFF0001, std::pair( 0x2000000, SignalPath{ 1, 1, 3 } ) } }; + auto idleCollectionScheme = std::make_shared( "IdleCollectionScheme", + "DM1", + startTimeInNearFuture, + stopTimeInDistantFuture, + signals1, + std::vector{}, + partialSignalIDLookup1 ); + + std::vector signals2 = { SignalCollectionInfo{ 0xFFFF0002 }, + SignalCollectionInfo{ 0xFFFF0003 } }; + ICollectionScheme::PartialSignalIDLookup partialSignalIDLookup2{ + { 0xFFFF0002, std::pair( 0x2000001, SignalPath{ 1, 2, 5 } ) }, + { 0xFFFF0003, std::pair( 0x2000001, SignalPath{ 1, 1, 3 } ) } }; + auto enabledCollectionScheme = std::make_shared( "EnabledCollectionScheme", + "DM1", + startTimeInThePast, + stopTimeInDistantFuture, + signals2, + std::vector{}, + partialSignalIDLookup2 ); + + mCollectionSchemeManager.onCollectionSchemeUpdate( std::make_shared( + std::vector{ idleCollectionScheme, enabledCollectionScheme } ) ); + + WAIT_ASSERT_EQ( mReceivedInspectionMatrices.size(), 2U ); + + // Repeat the same collection schemes, but modify the signals for the idle one. This is to make + // sure that later when CollectionSchemeManager enables this collection scheme, it will use the + // new signal IDs. + signals1 = { SignalCollectionInfo{ 0xFFFF0010 }, SignalCollectionInfo{ 0xFFFF0011 } }; + partialSignalIDLookup1 = { { 0xFFFF0010, std::pair( 0x2000000, SignalPath{ 1, 2, 5 } ) }, + { 0xFFFF0011, std::pair( 0x2000000, SignalPath{ 1, 1, 3 } ) } }; + idleCollectionScheme = std::make_shared( "IdleCollectionScheme", + "DM1", + startTimeInNearFuture, + stopTimeInDistantFuture, + signals1, + std::vector{}, + partialSignalIDLookup1 ); + + enabledCollectionScheme = std::make_shared( "EnabledCollectionScheme", + "DM1", + startTimeInThePast, + stopTimeInDistantFuture, + signals2, + std::vector{}, + partialSignalIDLookup2 ); + mCollectionSchemeManager.onCollectionSchemeUpdate( std::make_shared( + std::vector{ enabledCollectionScheme, idleCollectionScheme } ) ); + + WAIT_ASSERT_EQ( mReceivedInspectionMatrices.size(), 3U ); + + // Now send the same collection schemes, but slightly modified. In general collection schemes + // coming from the Cloud with the same ID should be considered the same. But for complex data + // some the partial signal IDs are generated by us and they can change when we ingest a new + // message from the Cloud. + idleCollectionScheme = + std::make_shared( "IdleCollectionScheme", + "DM1", + // Also make the idle collection scheme become enabled + startTimeInNearFuture, + stopTimeInDistantFuture, + signals1, + std::vector{}, + partialSignalIDLookup1 ); + + signals2 = { SignalCollectionInfo{ 0xFFFF0012 }, SignalCollectionInfo{ 0xFFFF0013 } }; + partialSignalIDLookup2 = { { 0xFFFF0012, std::pair( 0x2000001, SignalPath{ 1, 2, 5 } ) }, + { 0xFFFF0013, std::pair( 0x2000001, SignalPath{ 1, 1, 3 } ) } }; + enabledCollectionScheme = std::make_shared( "EnabledCollectionScheme", + "DM1", + startTimeInThePast, + stopTimeInDistantFuture, + signals2, + std::vector{}, + partialSignalIDLookup2 ); + mCollectionSchemeManager.onCollectionSchemeUpdate( std::make_shared( + std::vector{ enabledCollectionScheme, idleCollectionScheme } ) ); + + WAIT_ASSERT_EQ( mReceivedInspectionMatrices.size(), 4U ); + + auto lastReceivedInspectionMatrix = mReceivedInspectionMatrices.at( 3 ); + ASSERT_EQ( lastReceivedInspectionMatrix->conditions.size(), 2U ); + + ASSERT_EQ( getSignalIdsFromCondition( lastReceivedInspectionMatrix->conditions.at( 0 ) ), + ( std::vector{ 0xFFFF0012, 0xFFFF0013 } ) ); + ASSERT_EQ( getSignalIdsFromCondition( lastReceivedInspectionMatrix->conditions.at( 1 ) ), + ( std::vector{ 0xFFFF0010, 0xFFFF0011 } ) ); } +#endif } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/CredentialsTest.cpp b/test/unit/CredentialsTest.cpp index 3c482e14..6e21bb03 100644 --- a/test/unit/CredentialsTest.cpp +++ b/test/unit/CredentialsTest.cpp @@ -44,9 +44,9 @@ class CrtCredentialsProviderMock : public Aws::Crt::Auth::ICredentialsProvider MOCK_METHOD( bool, GetCredentials, ( const Aws::Crt::Auth::OnCredentialsResolved &onCredentialsResolved ), - ( const ) ); - MOCK_METHOD( aws_credentials_provider *, GetUnderlyingHandle, (), ( const, noexcept ) ); - MOCK_METHOD( bool, IsValid, (), ( const, noexcept ) ); + ( const, override ) ); + MOCK_METHOD( aws_credentials_provider *, GetUnderlyingHandle, (), ( const, noexcept, override ) ); + MOCK_METHOD( bool, IsValid, (), ( const, noexcept, override ) ); }; class CredentialsTest : public ::testing::Test diff --git a/test/unit/CustomDataSourceTest.cpp b/test/unit/CustomDataSourceTest.cpp index 954b9367..640891ee 100644 --- a/test/unit/CustomDataSourceTest.cpp +++ b/test/unit/CustomDataSourceTest.cpp @@ -35,7 +35,7 @@ using ::testing::SaveArgPointee; class CustomDataSourceTestImplementation : public CustomDataSource { public: - MOCK_METHOD( (void), pollData, () ); + MOCK_METHOD( (void), pollData, (), ( override ) ); void setPollInternalIntervalMs( uint32_t pollIntervalMs ) diff --git a/test/unit/DataSenderIonWriterTest.cpp b/test/unit/DataSenderIonWriterTest.cpp index 9c9bc1fd..d71e4cde 100644 --- a/test/unit/DataSenderIonWriterTest.cpp +++ b/test/unit/DataSenderIonWriterTest.cpp @@ -41,11 +41,11 @@ class DataSenderIonWriterTest : public ::testing::Test dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["ros2"]["SignalInfoFor1234Name:TypeEncoded"].mSignalId = 1234; - triggeredCollectionSchemeData = std::make_shared(); - triggeredCollectionSchemeData->metadata.decoderID = "TESTDECODERID"; - triggeredCollectionSchemeData->metadata.collectionSchemeID = "TESTCOLLECTIONSCHEME"; - triggeredCollectionSchemeData->triggerTime = 1000000; - triggeredCollectionSchemeData->eventID = 579; + triggeredVisionSystemData = std::make_shared(); + triggeredVisionSystemData->metadata.decoderID = "TESTDECODERID"; + triggeredVisionSystemData->metadata.collectionSchemeID = "TESTCOLLECTIONSCHEME"; + triggeredVisionSystemData->triggerTime = 1000000; + triggeredVisionSystemData->eventID = 579; bufferManager = std::make_shared>( RawData::BufferManagerConfig::create().get() ); @@ -56,7 +56,7 @@ class DataSenderIonWriterTest : public ::testing::Test ionWriter = std::make_unique( bufferManager, "DemoVehicle" ); ionWriter->onChangeOfActiveDictionary( dictionary, VehicleDataSourceProtocol::COMPLEX_DATA ); - ionWriter->setupVehicleData( triggeredCollectionSchemeData ); + ionWriter->setupVehicleData( triggeredVisionSystemData ); } void @@ -65,7 +65,7 @@ class DataSenderIonWriterTest : public ::testing::Test } std::shared_ptr dictionary; - std::shared_ptr triggeredCollectionSchemeData; + std::shared_ptr triggeredVisionSystemData; std::shared_ptr> bufferManager; RawData::SignalUpdateConfig signalConfig; std::unique_ptr ionWriter; @@ -80,7 +80,7 @@ TEST_F( DataSenderIonWriterTest, StreamOutputIonAfterSeekTheSame ) for ( int i = 0; i < 3; i++ ) { ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::COMPLEX_SIGNAL ) ); } auto estimatedSize = ionWriter->getEstimatedSizeInBytes(); auto stream = ionWriter->getStreambufBuilder()->build(); @@ -91,7 +91,7 @@ TEST_F( DataSenderIonWriterTest, StreamOutputIonAfterSeekTheSame ) EXPECT_THAT( firstIterationString, HasSubstr( "collection_event_time" ) ); EXPECT_THAT( firstIterationString, HasSubstr( "signal_name" ) ); - EXPECT_THAT( firstIterationString, HasSubstr( triggeredCollectionSchemeData->metadata.collectionSchemeID ) ); + EXPECT_THAT( firstIterationString, HasSubstr( triggeredVisionSystemData->metadata.collectionSchemeID ) ); EXPECT_THAT( firstIterationString, HasSubstr( "SignalInfoFor1234Name" ) ); // for better analysis create file @@ -131,7 +131,7 @@ TEST_F( DataSenderIonWriterTest, StreamOutputIonAfterAbsoluteSeek ) for ( int i = 0; i < 3; i++ ) { ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::COMPLEX_SIGNAL ) ); } auto estimatedSize = ionWriter->getEstimatedSizeInBytes(); auto stream = ionWriter->getStreambufBuilder()->build(); @@ -143,7 +143,7 @@ TEST_F( DataSenderIonWriterTest, StreamOutputIonAfterAbsoluteSeek ) EXPECT_THAT( firstIterationString, HasSubstr( "collection_event_time" ) ); EXPECT_THAT( firstIterationString, HasSubstr( "signal_name" ) ); - EXPECT_THAT( firstIterationString, HasSubstr( triggeredCollectionSchemeData->metadata.collectionSchemeID ) ); + EXPECT_THAT( firstIterationString, HasSubstr( triggeredVisionSystemData->metadata.collectionSchemeID ) ); EXPECT_THAT( firstIterationString, HasSubstr( "SignalInfoFor1234Name" ) ); // for better analysis create file @@ -226,7 +226,7 @@ TEST_F( DataSenderIonWriterTest, StreamOutputHugeIon ) for ( int i = 0; i < 1000; i++ ) { ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 33444444, handle, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + static_cast( signalConfig.typeId ), 33444444, handle, SignalType::COMPLEX_SIGNAL ) ); } auto stream = ionWriter->getStreambufBuilder()->build(); { @@ -251,7 +251,7 @@ TEST_F( DataSenderIonWriterTest, StreamTellgStream ) for ( int i = 0; i < 3; i++ ) { ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + static_cast( signalConfig.typeId ), 3000000 + i, handle, SignalType::COMPLEX_SIGNAL ) ); } auto stream = ionWriter->getStreambufBuilder()->build(); @@ -306,10 +306,10 @@ TEST_F( DataSenderIonWriterTest, ReturnNullStreamWhenAllDataIsDeleted ) handle2, RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_SELECTED_FOR_UPLOAD ) ); - ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000000, handle1, SignalType::RAW_DATA_BUFFER_HANDLE ) ); - ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000001, handle2, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + ionWriter->append( + CollectedSignal( static_cast( signalConfig.typeId ), 3000000, handle1, SignalType::COMPLEX_SIGNAL ) ); + ionWriter->append( + CollectedSignal( static_cast( signalConfig.typeId ), 3000001, handle2, SignalType::COMPLEX_SIGNAL ) ); // Before we get the stream, make the buffer manager delete the data ASSERT_TRUE( bufferManager->decreaseHandleUsageHint( @@ -351,8 +351,8 @@ TEST_F( DataSenderIonWriterTest, HandleUsageIsUpdatedWhenStreamIsCreated ) RawData::BufferHandleUsageStage::COLLECTION_INSPECTION_ENGINE_SELECTED_FOR_UPLOAD ) ) .Times( 1 ); } - ionWriter->append( CollectedSignal( - static_cast( signalConfig.typeId ), 3000000, handle, SignalType::RAW_DATA_BUFFER_HANDLE ) ); + ionWriter->append( + CollectedSignal( static_cast( signalConfig.typeId ), 3000000, handle, SignalType::COMPLEX_SIGNAL ) ); // Now then the stream is really created the usage should be set as uploading, which should prevent data // to be deleted. diff --git a/test/unit/DataSenderManagerTest.cpp b/test/unit/DataSenderManagerTest.cpp index 0ab2e289..422a0bc2 100644 --- a/test/unit/DataSenderManagerTest.cpp +++ b/test/unit/DataSenderManagerTest.cpp @@ -6,15 +6,21 @@ #include "CANInterfaceIDTranslator.h" #include "CacheAndPersist.h" #include "CollectionInspectionAPITypes.h" +#include "DataSenderProtoWriter.h" +#include "DataSenderTypes.h" #include "IConnectionTypes.h" -#include "ISender.h" #include "OBDDataTypes.h" -#include "PayloadManagerMock.h" +#include "PayloadManager.h" +#include "RawDataManager.h" #include "SenderMock.h" #include "SignalTypes.h" +#include "TelemetryDataSender.h" +#include "Testing.h" #include "vehicle_data.pb.h" #include +#include #include +#include #include #include #include @@ -22,6 +28,8 @@ #include #include #include +#include +#include #include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -29,13 +37,14 @@ #include "DataSenderIonWriterMock.h" #include "ICollectionScheme.h" #include "ICollectionSchemeList.h" +#include "QueueTypes.h" +#include "S3Sender.h" #include "S3SenderMock.h" #include "StreambufBuilder.h" #include "StringbufBuilder.h" -#include +#include "VisionSystemDataSender.h" #include #include -#include #endif namespace Aws @@ -44,9 +53,11 @@ namespace IoTFleetWise { using ::testing::_; -using ::testing::DoAll; +using ::testing::AnyNumber; +using ::testing::AtLeast; using ::testing::Gt; using ::testing::Invoke; +using ::testing::InvokeArgument; using ::testing::Return; using ::testing::SetArgReferee; using ::testing::StrictMock; @@ -59,6 +70,10 @@ class DataSenderManagerTest : public ::testing::Test void SetUp() override { + std::shared_ptr mRawDataBufferManager; + mPersistency = createCacheAndPersist(); + mPayloadManager = std::make_shared( mPersistency ); + mTriggeredCollectionSchemeData = std::make_shared(); mTriggeredCollectionSchemeData->metadata.decoderID = "TESTDECODERID"; mTriggeredCollectionSchemeData->metadata.collectionSchemeID = "TESTCOLLECTIONSCHEME"; @@ -68,66 +83,92 @@ class DataSenderManagerTest : public ::testing::Test mCANIDTranslator.add( "can123" ); mMqttSender = std::make_shared>(); - mPayloadManager = std::make_shared>(); + EXPECT_CALL( *mMqttSender, isAlive() ).Times( AnyNumber() ).WillRepeatedly( Return( true ) ); + EXPECT_CALL( *mMqttSender, getMaxSendSize() ) + .Times( AnyNumber() ) + .WillRepeatedly( Return( MAXIMUM_PAYLOAD_SIZE ) ); + ON_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ) + .WillByDefault( InvokeArgument<2>( ConnectivityError::Success ) ); + + auto mProtoWriter = std::make_shared( mCANIDTranslator, mRawDataBufferManager ); + auto telemetryDataSender = std::make_shared( + mMqttSender, mProtoWriter, mPayloadAdaptionConfigUncompressed, mPayloadAdaptionConfigCompressed ); + std::unordered_map> dataSenders; + dataSenders[SenderDataType::TELEMETRY] = std::move( telemetryDataSender ); #ifdef FWE_FEATURE_VISION_SYSTEM_DATA + mTriggeredVisionSystemData = std::make_shared(); + mTriggeredVisionSystemData->metadata.decoderID = "TESTDECODERID"; + mTriggeredVisionSystemData->metadata.collectionSchemeID = "TESTCOLLECTIONSCHEME"; + mTriggeredVisionSystemData->triggerTime = 1000000; + mTriggeredVisionSystemData->eventID = 579; + mS3Sender = std::make_shared>(); mIonWriter = std::make_shared>(); + mUploadedS3Objects = std::make_shared( 100, "Uploaded S3 Objects" ); mActiveCollectionSchemes = std::make_shared(); + mVisionSystemDataSender = + std::make_shared( mUploadedS3Objects, mS3Sender, mIonWriter, "" ); + dataSenders[SenderDataType::VISION_SYSTEM] = mVisionSystemDataSender; #endif - mDataSenderManager = std::make_unique( mMqttSender, - mPayloadManager, - mCANIDTranslator, - mTransmitThreshold -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - mS3Sender, - mIonWriter, - "" -#endif - ); + mDataSenderManager = + std::make_unique( std::move( dataSenders ), mMqttSender, mPayloadManager ); } void TearDown() override { } - - void - processCollectedData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr ) - { - mDataSenderManager->processCollectedData( triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - nullptr -#endif - ); - } - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA void - processCollectedData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr, - std::function reportUploadCallback ) + processCollectedData( std::shared_ptr data ) { - mDataSenderManager->processCollectedData( triggeredCollectionSchemeDataPtr, reportUploadCallback ); + mDataSenderManager->processData( std::const_pointer_cast( data ) ); } -#endif protected: - unsigned mTransmitThreshold{ 5 }; // max number of messages that can be sent to cloud at one time + static constexpr unsigned MAXIMUM_PAYLOAD_SIZE = 400; + static constexpr unsigned CAN_DATA_SIZE = 8; + PayloadAdaptionConfig mPayloadAdaptionConfigUncompressed{ 80, 70, 90, 10 }; + PayloadAdaptionConfig mPayloadAdaptionConfigCompressed{ 80, 70, 90, 10 }; unsigned mCanChannelID{ 0 }; std::shared_ptr mTriggeredCollectionSchemeData; std::shared_ptr> mMqttSender; - std::shared_ptr> mPayloadManager; + std::shared_ptr mPersistency; + std::shared_ptr mPayloadManager; CANInterfaceIDTranslator mCANIDTranslator; std::unique_ptr mDataSenderManager; + std::shared_ptr mProtoWriter; #ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::shared_ptr mTriggeredVisionSystemData; + std::shared_ptr mVisionSystemDataSender; std::shared_ptr> mS3Sender; std::shared_ptr> mIonWriter; std::shared_ptr mActiveCollectionSchemes; + std::shared_ptr mUploadedS3Objects; #endif }; +TEST_F( DataSenderManagerTest, senderDataTypeToString ) +{ + EXPECT_EQ( senderDataTypeToString( SenderDataType::TELEMETRY ), "Telemetry" ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + EXPECT_EQ( senderDataTypeToString( SenderDataType::VISION_SYSTEM ), "VisionSystem" ); +#endif + EXPECT_EQ( senderDataTypeToString( static_cast( -1 ) ), "" ); +} + +TEST_F( DataSenderManagerTest, stringToSenderDataType ) +{ + SenderDataType output; + EXPECT_TRUE( stringToSenderDataType( "Telemetry", output ) ); + EXPECT_EQ( output, SenderDataType::TELEMETRY ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + EXPECT_TRUE( stringToSenderDataType( "VisionSystem", output ) ); + EXPECT_EQ( output, SenderDataType::VISION_SYSTEM ); +#endif + EXPECT_FALSE( stringToSenderDataType( "Invalid", output ) ); +} + TEST_F( DataSenderManagerTest, ProcessEmptyData ) { EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); @@ -141,22 +182,21 @@ TEST_F( DataSenderManagerTest, ProcessSingleSignal ) auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); mTriggeredCollectionSchemeData->signals.push_back( signal1 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 1 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); @@ -174,22 +214,21 @@ TEST_F( DataSenderManagerTest, ProcessMultipleSignals ) mTriggeredCollectionSchemeData->signals.push_back( signal2 ); mTriggeredCollectionSchemeData->signals.push_back( signal3 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 3 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); @@ -205,6 +244,15 @@ TEST_F( DataSenderManagerTest, ProcessMultipleSignals ) TEST_F( DataSenderManagerTest, ProcessMultipleSignalsBeyondTransmitThreshold ) { std::vector signals = { CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), + CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ), @@ -217,34 +265,32 @@ TEST_F( DataSenderManagerTest, ProcessMultipleSignalsBeyondTransmitThreshold ) EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) .Times( 2 ) - .WillRepeatedly( Return( ConnectivityError::Success ) ); + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 2 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - - collectionSchemeParams = sentBufferData[1].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); - ASSERT_EQ( vehicleData.captured_signals_size(), 5 ); + EXPECT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + EXPECT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + EXPECT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + EXPECT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + ASSERT_EQ( vehicleData.captured_signals_size(), 14 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + EXPECT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + EXPECT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + EXPECT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + EXPECT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 4 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); @@ -253,125 +299,129 @@ TEST_F( DataSenderManagerTest, ProcessMultipleSignalsBeyondTransmitThreshold ) TEST_F( DataSenderManagerTest, ProcessSingleCanFrame ) { std::array canBuf1 = { 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; - auto canFrame1 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ); + auto canFrame1 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ); mTriggeredCollectionSchemeData->canFrames.push_back( canFrame1 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 1 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); ASSERT_EQ( vehicleData.can_frames()[0].message_id(), canFrame1.frameID ); ASSERT_EQ( vehicleData.can_frames()[0].interface_id(), "can123" ); - ASSERT_EQ( vehicleData.can_frames()[0].byte_values(), std::string( canBuf1.begin(), canBuf1.end() ) ); + ASSERT_EQ( vehicleData.can_frames()[0].byte_values(), + std::string( canBuf1.begin(), canBuf1.begin() + CAN_DATA_SIZE ) ); } TEST_F( DataSenderManagerTest, ProcessMultipleCanFrames ) { std::array canBuf1 = { 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; - auto canFrame1 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ); + auto canFrame1 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ); std::array canBuf2 = { 0xBA, 0xAD, 0xAF, 0xFE, 0x0, 0x0, 0x0, 0x0 }; - auto canFrame2 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf2, sizeof( canBuf2 ) ); + auto canFrame2 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf2, CAN_DATA_SIZE ); std::array canBuf3 = { 0xCA, 0xFE, 0xF0, 0x0D, 0x0, 0x0, 0x0, 0x0 }; - auto canFrame3 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf3, sizeof( canBuf3 ) ); + auto canFrame3 = CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf3, CAN_DATA_SIZE ); mTriggeredCollectionSchemeData->canFrames.push_back( canFrame1 ); mTriggeredCollectionSchemeData->canFrames.push_back( canFrame2 ); mTriggeredCollectionSchemeData->canFrames.push_back( canFrame3 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 3 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); ASSERT_EQ( vehicleData.can_frames()[0].message_id(), canFrame1.frameID ); ASSERT_EQ( vehicleData.can_frames()[0].interface_id(), "can123" ); - ASSERT_EQ( vehicleData.can_frames()[0].byte_values(), std::string( canBuf1.begin(), canBuf1.end() ) ); + ASSERT_EQ( vehicleData.can_frames()[0].byte_values(), + std::string( canBuf1.begin(), canBuf1.begin() + CAN_DATA_SIZE ) ); ASSERT_EQ( vehicleData.can_frames()[1].message_id(), canFrame2.frameID ); ASSERT_EQ( vehicleData.can_frames()[1].interface_id(), "can123" ); - ASSERT_EQ( vehicleData.can_frames()[1].byte_values(), std::string( canBuf2.begin(), canBuf2.end() ) ); + ASSERT_EQ( vehicleData.can_frames()[1].byte_values(), + std::string( canBuf2.begin(), canBuf2.begin() + CAN_DATA_SIZE ) ); ASSERT_EQ( vehicleData.can_frames()[2].message_id(), canFrame3.frameID ); ASSERT_EQ( vehicleData.can_frames()[2].interface_id(), "can123" ); - ASSERT_EQ( vehicleData.can_frames()[2].byte_values(), std::string( canBuf3.begin(), canBuf3.end() ) ); + ASSERT_EQ( vehicleData.can_frames()[2].byte_values(), + std::string( canBuf3.begin(), canBuf3.begin() + CAN_DATA_SIZE ) ); } TEST_F( DataSenderManagerTest, ProcessMultipleCanFramesBeyondTransmitThreshold ) { std::array canBuf1 = { 0xDE, 0xAD, 0xBE, 0xEF, 0x0, 0x0, 0x0, 0x0 }; std::vector canFrames = { - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ), - CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, sizeof( canBuf1 ) ) }; + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ), + CollectedCanRawFrame( 0x380, mCanChannelID, 789654, canBuf1, CAN_DATA_SIZE ) }; mTriggeredCollectionSchemeData->canFrames = canFrames; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) - .Times( 2 ) - .WillRepeatedly( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 2 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 2 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - - collectionSchemeParams = sentBufferData[1].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); - ASSERT_EQ( vehicleData.can_frames_size(), 5 ); + ASSERT_EQ( vehicleData.can_frames_size(), 10 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); - ASSERT_EQ( vehicleData.can_frames_size(), 4 ); + ASSERT_EQ( vehicleData.can_frames_size(), 5 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); } @@ -383,22 +433,21 @@ TEST_F( DataSenderManagerTest, ProcessSingleDtcCode ) dtcInfo.mDTCCodes.emplace_back( "P0143" ); mTriggeredCollectionSchemeData->mDTCInfo = dtcInfo; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_TRUE( vehicleData.has_dtc_data() ); @@ -416,22 +465,21 @@ TEST_F( DataSenderManagerTest, ProcessMultipleDtcCodes ) dtcInfo.mDTCCodes.emplace_back( "C0196" ); mTriggeredCollectionSchemeData->mDTCInfo = dtcInfo; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_TRUE( vehicleData.has_dtc_data() ); @@ -443,8 +491,16 @@ TEST_F( DataSenderManagerTest, ProcessMultipleDtcCodes ) TEST_F( DataSenderManagerTest, ProcessMultipleDtcCodesBeyondTransmitThreshold ) { - std::vector dtcCodes = { - "P0143", "C0196", "U0148", "B0148", "C0148", "C0149", "C0150", "C0151", "C0152" }; + std::vector dtcCodes = { "P0143_________________________", + "C0196_________________________", + "U0148_________________________", + "B0148_________________________", + "C0148_________________________", + "C0149_________________________", + "C0150_________________________", + "C0151_________________________", + "C0152_________________________", + "C0153_________________________" }; DTCInfo dtcInfo; dtcInfo.mSID = SID::STORED_DTC; @@ -452,35 +508,37 @@ TEST_F( DataSenderManagerTest, ProcessMultipleDtcCodesBeyondTransmitThreshold ) dtcInfo.mDTCCodes = dtcCodes; mTriggeredCollectionSchemeData->mDTCInfo = dtcInfo; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) - .Times( 2 ) - .WillRepeatedly( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 2 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 2 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_TRUE( vehicleData.has_dtc_data() ); - ASSERT_EQ( vehicleData.dtc_data().active_dtc_codes_size(), 5 ); + ASSERT_EQ( vehicleData.dtc_data().active_dtc_codes_size(), 9 ); ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_TRUE( vehicleData.has_dtc_data() ); - ASSERT_EQ( vehicleData.dtc_data().active_dtc_codes_size(), 4 ); + ASSERT_EQ( vehicleData.dtc_data().active_dtc_codes_size(), 1 ); } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -489,22 +547,21 @@ TEST_F( DataSenderManagerTest, ProcessSingleUploadedS3Object ) auto uploadedS3Object1 = UploadedS3Object{ "uploaded/object/key1", UploadedS3ObjectDataFormat::Cdr }; mTriggeredCollectionSchemeData->uploadedS3Objects.push_back( uploadedS3Object1 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); @@ -523,22 +580,21 @@ TEST_F( DataSenderManagerTest, ProcessMultipleUploadedS3Objects ) mTriggeredCollectionSchemeData->uploadedS3Objects.push_back( uploadedS3Object2 ); mTriggeredCollectionSchemeData->uploadedS3Objects.push_back( uploadedS3Object3 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); @@ -564,63 +620,74 @@ TEST_F( DataSenderManagerTest, ProcessMultipleUploadedS3ObjectsBeyondTransmitThr UploadedS3Object{ "uploaded/object/key7", UploadedS3ObjectDataFormat::Cdr }, UploadedS3Object{ "uploaded/object/key8", UploadedS3ObjectDataFormat::Cdr }, UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, + UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, + UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, + UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, + UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, + UploadedS3Object{ "uploaded/object/key9", UploadedS3ObjectDataFormat::Cdr }, }; mTriggeredCollectionSchemeData->uploadedS3Objects = uploadedS3Objects; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) - .Times( 2 ) - .WillRepeatedly( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 2 ); processCollectedData( mTriggeredCollectionSchemeData ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 2 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - - collectionSchemeParams = sentBufferData[1].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); - Schemas::VehicleDataMsg::VehicleData vehicleData; ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); - ASSERT_EQ( vehicleData.s3_objects_size(), 5 ); + ASSERT_EQ( vehicleData.s3_objects_size(), 11 ); ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + ASSERT_EQ( vehicleData.captured_signals_size(), 0 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); ASSERT_FALSE( vehicleData.has_dtc_data() ); - ASSERT_EQ( vehicleData.s3_objects_size(), 4 ); + ASSERT_EQ( vehicleData.s3_objects_size(), 3 ); } TEST_F( DataSenderManagerTest, ProcessRawDataSignalNoActiveCampaigns ) { - auto signal1 = CollectedSignal( 1234, 789654, 10000, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + auto signal1 = CollectedSignal( 1234, 789654, 10000, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal1 ); EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); EXPECT_CALL( *mIonWriter, setupVehicleData( _ ) ).Times( 1 ); EXPECT_CALL( *mIonWriter, mockedAppend( _ ) ).Times( 1 ); EXPECT_CALL( *mIonWriter, getStreambufBuilder() ).Times( 0 ); - processCollectedData( mTriggeredCollectionSchemeData ); + processCollectedData( mTriggeredVisionSystemData ); +} + +TEST_F( DataSenderManagerTest, ProcessVisionSystemDataWithoutRawData ) +{ + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); + EXPECT_CALL( *mIonWriter, setupVehicleData( _ ) ).Times( 0 ); + EXPECT_CALL( *mIonWriter, mockedAppend( _ ) ).Times( 0 ); + EXPECT_CALL( *mIonWriter, getStreambufBuilder() ).Times( 0 ); + + processCollectedData( mTriggeredVisionSystemData ); } TEST_F( DataSenderManagerTest, ProcessSingleRawDataSignal ) { - auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal1 ); auto s3UploadMetadata = S3UploadMetadata(); s3UploadMetadata.bucketName = "BucketName"; @@ -640,34 +707,30 @@ TEST_F( DataSenderManagerTest, ProcessSingleRawDataSignal ) std::unique_ptr sentStream; EXPECT_CALL( *mS3Sender, sendStream( _, s3UploadMetadata, std::string( "s3/prefix/raw-data/579-1000000.10n" ), _ ) ) - // Can't use DoAll(InvokeArgument, Return) here: https://stackoverflow.com/a/70886530 .WillOnce( WithArgs<0, 3>( [&sentStream]( std::unique_ptr streambufBuilder, - std::function resultCallback ) { + S3Sender::ResultCallback resultCallback ) { sentStream = std::move( streambufBuilder->build() ); - resultCallback( true ); - return ConnectivityError::Success; + resultCallback( ConnectivityError::Success, nullptr ); } ) ); - TriggeredCollectionSchemeDataPtr reportedCollectionSchemeData = nullptr; - mDataSenderManager->onChangeCollectionSchemeList( mActiveCollectionSchemes ); - processCollectedData( - mTriggeredCollectionSchemeData, - [&reportedCollectionSchemeData]( TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData ) { - reportedCollectionSchemeData = triggeredCollectionSchemeData; - } ); + mVisionSystemDataSender->onChangeCollectionSchemeList( mActiveCollectionSchemes ); + processCollectedData( mTriggeredVisionSystemData ); ASSERT_EQ( mIonWriter->mSignals.size(), 1 ); ASSERT_EQ( mIonWriter->mSignals[0].signalID, 1234 ); ASSERT_EQ( mIonWriter->mSignals[0].value.value.uint32Val, 888999U ); + std::shared_ptr senderData; + ASSERT_TRUE( mUploadedS3Objects->pop( senderData ) ); + auto reportedCollectionSchemeData = std::dynamic_pointer_cast( senderData ); ASSERT_NE( reportedCollectionSchemeData, nullptr ); - ASSERT_EQ( reportedCollectionSchemeData->eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( reportedCollectionSchemeData->triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.compress, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.persist, mTriggeredCollectionSchemeData->metadata.persist ); + ASSERT_EQ( reportedCollectionSchemeData->eventID, mTriggeredVisionSystemData->eventID ); + ASSERT_EQ( reportedCollectionSchemeData->triggerTime, mTriggeredVisionSystemData->triggerTime ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.compress, mTriggeredVisionSystemData->metadata.compress ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.persist, mTriggeredVisionSystemData->metadata.persist ); ASSERT_EQ( reportedCollectionSchemeData->metadata.collectionSchemeID, - mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.decoderID, mTriggeredCollectionSchemeData->metadata.decoderID ); + mTriggeredVisionSystemData->metadata.collectionSchemeID ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.decoderID, mTriggeredVisionSystemData->metadata.decoderID ); ASSERT_FALSE( reportedCollectionSchemeData->mDTCInfo.hasItems() ); ASSERT_EQ( reportedCollectionSchemeData->signals.size(), 0 ); @@ -683,10 +746,10 @@ TEST_F( DataSenderManagerTest, ProcessSingleRawDataSignal ) TEST_F( DataSenderManagerTest, ProcessMultipleRawDataSignals ) { - auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal1 ); - auto signal2 = CollectedSignal( 5678, 789987, 889000, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal2 ); + auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal1 ); + auto signal2 = CollectedSignal( 5678, 789987, 889000, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal2 ); auto s3UploadMetadata = S3UploadMetadata(); s3UploadMetadata.bucketName = "BucketName"; @@ -705,18 +768,12 @@ TEST_F( DataSenderManagerTest, ProcessMultipleRawDataSignals ) } ); EXPECT_CALL( *mS3Sender, sendStream( _, s3UploadMetadata, std::string( "s3/prefix/raw-data/579-1000000.10n" ), _ ) ) // Can't use DoAll(InvokeArgument, Return) here: https://stackoverflow.com/a/70886530 - .WillOnce( WithArg<3>( []( std::function resultCallback ) { - resultCallback( true ); - return ConnectivityError::Success; + .WillOnce( WithArg<3>( []( S3Sender::ResultCallback resultCallback ) { + resultCallback( ConnectivityError::Success, nullptr ); } ) ); - TriggeredCollectionSchemeDataPtr reportedCollectionSchemeData = nullptr; - mDataSenderManager->onChangeCollectionSchemeList( mActiveCollectionSchemes ); - processCollectedData( - mTriggeredCollectionSchemeData, - [&reportedCollectionSchemeData]( TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData ) { - reportedCollectionSchemeData = triggeredCollectionSchemeData; - } ); + mVisionSystemDataSender->onChangeCollectionSchemeList( mActiveCollectionSchemes ); + processCollectedData( mTriggeredVisionSystemData ); ASSERT_EQ( mIonWriter->mSignals.size(), 2 ); ASSERT_EQ( mIonWriter->mSignals[0].signalID, 1234 ); @@ -724,14 +781,17 @@ TEST_F( DataSenderManagerTest, ProcessMultipleRawDataSignals ) ASSERT_EQ( mIonWriter->mSignals[1].signalID, 5678 ); ASSERT_EQ( mIonWriter->mSignals[1].value.value.uint32Val, 889000U ); + std::shared_ptr senderData; + ASSERT_TRUE( mUploadedS3Objects->pop( senderData ) ); + auto reportedCollectionSchemeData = std::dynamic_pointer_cast( senderData ); ASSERT_NE( reportedCollectionSchemeData, nullptr ); - ASSERT_EQ( reportedCollectionSchemeData->eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( reportedCollectionSchemeData->triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.compress, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.persist, mTriggeredCollectionSchemeData->metadata.persist ); + ASSERT_EQ( reportedCollectionSchemeData->eventID, mTriggeredVisionSystemData->eventID ); + ASSERT_EQ( reportedCollectionSchemeData->triggerTime, mTriggeredVisionSystemData->triggerTime ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.compress, mTriggeredVisionSystemData->metadata.compress ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.persist, mTriggeredVisionSystemData->metadata.persist ); ASSERT_EQ( reportedCollectionSchemeData->metadata.collectionSchemeID, - mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); - ASSERT_EQ( reportedCollectionSchemeData->metadata.decoderID, mTriggeredCollectionSchemeData->metadata.decoderID ); + mTriggeredVisionSystemData->metadata.collectionSchemeID ); + ASSERT_EQ( reportedCollectionSchemeData->metadata.decoderID, mTriggeredVisionSystemData->metadata.decoderID ); ASSERT_FALSE( reportedCollectionSchemeData->mDTCInfo.hasItems() ); ASSERT_EQ( reportedCollectionSchemeData->signals.size(), 0 ); @@ -741,8 +801,8 @@ TEST_F( DataSenderManagerTest, ProcessMultipleRawDataSignals ) TEST_F( DataSenderManagerTest, ProcessRawDataSignalFailure ) { - auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal1 ); auto s3UploadMetadata = S3UploadMetadata(); s3UploadMetadata.bucketName = "BucketName"; @@ -761,24 +821,67 @@ TEST_F( DataSenderManagerTest, ProcessRawDataSignalFailure ) } ); EXPECT_CALL( *mS3Sender, sendStream( _, s3UploadMetadata, std::string( "s3/prefix/raw-data/579-1000000.10n" ), _ ) ) // Can't use DoAll(InvokeArgument, Return) here: https://stackoverflow.com/a/70886530 - .WillOnce( WithArg<3>( []( std::function resultCallback ) { - resultCallback( false ); - return ConnectivityError::Success; + .WillOnce( WithArg<3>( []( S3Sender::ResultCallback resultCallback ) { + resultCallback( ConnectivityError::TransmissionError, nullptr ); } ) ); - TriggeredCollectionSchemeDataPtr reportedCollectionSchemeData = nullptr; - mDataSenderManager->onChangeCollectionSchemeList( mActiveCollectionSchemes ); - processCollectedData( - mTriggeredCollectionSchemeData, - [&reportedCollectionSchemeData]( TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData ) { - reportedCollectionSchemeData = triggeredCollectionSchemeData; - } ); + mVisionSystemDataSender->onChangeCollectionSchemeList( mActiveCollectionSchemes ); + processCollectedData( mTriggeredVisionSystemData ); ASSERT_EQ( mIonWriter->mSignals.size(), 1 ); ASSERT_EQ( mIonWriter->mSignals[0].signalID, 1234 ); ASSERT_EQ( mIonWriter->mSignals[0].value.value.uint32Val, 888999U ); - ASSERT_EQ( reportedCollectionSchemeData, nullptr ); + std::shared_ptr senderData; + ASSERT_FALSE( mUploadedS3Objects->pop( senderData ) ); +} + +TEST_F( DataSenderManagerTest, ProcessRawDataSignalFailureWithPersistency ) +{ + mTriggeredVisionSystemData->metadata.persist = true; + + auto signal1 = CollectedSignal( 1234, 789654, 888999, SignalType::COMPLEX_SIGNAL ); + mTriggeredVisionSystemData->signals.push_back( signal1 ); + + auto s3UploadMetadata = S3UploadMetadata(); + s3UploadMetadata.bucketName = "BucketName"; + s3UploadMetadata.bucketOwner = "1234567890"; + s3UploadMetadata.prefix = "s3/prefix/raw-data/"; + s3UploadMetadata.region = "eu-central-1"; + ICollectionSchemePtr collectionScheme1 = + std::make_shared( "TESTCOLLECTIONSCHEME", "TESTDECODERID", 0, 10, s3UploadMetadata ); + mActiveCollectionSchemes->activeCollectionSchemes.push_back( collectionScheme1 ); + std::string payload = "fake ion file"; + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); + EXPECT_CALL( *mIonWriter, setupVehicleData( _ ) ).Times( 1 ); + EXPECT_CALL( *mIonWriter, mockedAppend( _ ) ).Times( 1 ); + EXPECT_CALL( *mIonWriter, getStreambufBuilder() ).WillOnce( [payload]() { + return std::move( std::make_unique( payload ) ); + } ); + EXPECT_CALL( *mS3Sender, sendStream( _, s3UploadMetadata, std::string( "s3/prefix/raw-data/579-1000000.10n" ), _ ) ) + // Can't use DoAll(InvokeArgument, Return) here: https://stackoverflow.com/a/70886530 + .WillOnce( WithArg<3>( [payload]( S3Sender::ResultCallback resultCallback ) { + // This should make the file to be persisted + resultCallback( ConnectivityError::TransmissionError, + std::make_unique( payload )->build() ); + } ) ); + + mVisionSystemDataSender->onChangeCollectionSchemeList( mActiveCollectionSchemes ); + processCollectedData( mTriggeredVisionSystemData ); + + ASSERT_EQ( mIonWriter->mSignals.size(), 1 ); + ASSERT_EQ( mIonWriter->mSignals[0].signalID, 1234 ); + ASSERT_EQ( mIonWriter->mSignals[0].value.value.uint32Val, 888999U ); + + std::shared_ptr senderData; + ASSERT_FALSE( mUploadedS3Objects->pop( senderData ) ); + + std::vector fileContent( payload.size() ); + ASSERT_EQ( + mPayloadManager->retrievePayload( fileContent.data(), payload.size(), "s3/prefix/raw-data/579-1000000.10n" ), + ErrorCode::SUCCESS ); + ASSERT_EQ( std::string( fileContent.begin(), fileContent.end() ), payload ); } TEST_F( DataSenderManagerTest, ProcessSingleSignalWithoutRawData ) @@ -797,58 +900,232 @@ TEST_F( DataSenderManagerTest, ProcessSingleSignalWithoutRawData ) std::make_shared( "TESTCOLLECTIONSCHEME", "TESTDECODERID", 0, 10, s3UploadMetadata ); mActiveCollectionSchemes->activeCollectionSchemes.push_back( collectionScheme1 ); - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); EXPECT_CALL( *mS3Sender, sendStream( _, _, _, _ ) ).Times( 0 ); - TriggeredCollectionSchemeDataPtr reportedCollectionSchemeData = nullptr; - mDataSenderManager->onChangeCollectionSchemeList( mActiveCollectionSchemes ); - processCollectedData( - mTriggeredCollectionSchemeData, - [&reportedCollectionSchemeData]( TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeData ) { - reportedCollectionSchemeData = triggeredCollectionSchemeData; - } ); + mVisionSystemDataSender->onChangeCollectionSchemeList( mActiveCollectionSchemes ); + processCollectedData( mTriggeredCollectionSchemeData ); - ASSERT_EQ( reportedCollectionSchemeData, nullptr ); + std::shared_ptr senderData; + ASSERT_FALSE( mUploadedS3Objects->pop( senderData ) ); ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); } #endif +TEST_F( DataSenderManagerTest, ProcessSingleSignalWithCompression ) +{ + auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); + mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + mTriggeredCollectionSchemeData->metadata.compress = true; + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); + + processCollectedData( mTriggeredCollectionSchemeData ); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + Schemas::VehicleDataMsg::VehicleData vehicleData; + std::string uncompressedData; + snappy::Uncompress( sentBufferData[0].data.c_str(), sentBufferData[0].data.size(), &uncompressedData ); + ASSERT_TRUE( vehicleData.ParseFromString( uncompressedData ) ); + + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + ASSERT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_EQ( vehicleData.can_frames_size(), 0 ); + ASSERT_FALSE( vehicleData.has_dtc_data() ); + + ASSERT_EQ( vehicleData.captured_signals()[0].signal_id(), signal1.signalID ); + ASSERT_EQ( vehicleData.captured_signals()[0].double_value(), signal1.value.value.doubleVal ); +} + TEST_F( DataSenderManagerTest, PersistencyNoFiles ) { - Json::Value files( Json::arrayValue ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); - EXPECT_CALL( *mPayloadManager, retrievePayloadMetadata( _ ) ) - .WillOnce( DoAll( SetArgReferee<0>( files ), Return( ErrorCode::SUCCESS ) ) ); + mDataSenderManager->checkAndSendRetrievedData(); +} + +TEST_F( DataSenderManagerTest, PersistencyUnsupportedFile ) +{ + std::string filename = "filename1.bin"; + std::string payload = "fake collection payload"; - EXPECT_CALL( *mMqttSender, sendFile( _, _, _ ) ).Times( 0 ); + Json::Value payloadMetadata; + payloadMetadata["filename"] = filename; + payloadMetadata["payloadSize"] = payload.size(); + + Json::Value metadata; + metadata["type"] = "SomeUnsupportedType"; + metadata["payload"] = payloadMetadata; + + mPayloadManager->storeData( + reinterpret_cast( payload.data() ), payload.size(), metadata, filename ); + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); mDataSenderManager->checkAndSendRetrievedData(); } -TEST_F( DataSenderManagerTest, ProcessSingleSignalWithCompression ) +TEST_F( DataSenderManagerTest, PersistencyMissingPayloadFile ) { + std::string filename = "filename1.bin"; + + Json::Value payloadMetadata; + payloadMetadata["filename"] = filename; + payloadMetadata["payloadSize"] = 10; + + Json::Value metadata; + metadata["type"] = senderDataTypeToString( SenderDataType::TELEMETRY ); + metadata["payload"] = payloadMetadata; + + mPersistency->addMetadata( metadata ); + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, _, _ ) ).Times( 0 ); + + mDataSenderManager->checkAndSendRetrievedData(); +} + +TEST_F( DataSenderManagerTest, PersistencyForTelemetryDisabled ) +{ + mTriggeredCollectionSchemeData->metadata.persist = false; auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); mTriggeredCollectionSchemeData->signals.push_back( signal1 ); - mTriggeredCollectionSchemeData->metadata.compress = true; - EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::TransmissionError ) ); + + processCollectedData( mTriggeredCollectionSchemeData ); + + mMqttSender->clearSentBufferData(); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); +} + +TEST_F( DataSenderManagerTest, PersistencyForTelemetryLegacyMetadata ) +{ + std::string filename = "12345-1234567890.bin"; + std::string payload = "fake collection payload"; + + Json::Value metadata; + metadata["filename"] = filename; + metadata["payloadSize"] = payload.size(); + metadata["compressionRequired"] = true; + + mPayloadManager->storeData( + reinterpret_cast( payload.data() ), payload.size(), metadata, filename ); + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ).Times( 1 ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + ASSERT_EQ( sentBufferData[0].data, payload ); + + // Ensure that there is no more data persisted + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); +} + +TEST_F( DataSenderManagerTest, PersistencyForTelemetrySingleFile ) +{ + mTriggeredCollectionSchemeData->metadata.persist = true; + auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); + mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::TransmissionError ) ); processCollectedData( mTriggeredCollectionSchemeData ); + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::Success ) ); + + mDataSenderManager->checkAndSendRetrievedData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); auto sentBufferData = mMqttSender->getSentBufferData(); - auto collectionSchemeParams = sentBufferData[0].collectionSchemeParams; - ASSERT_EQ( collectionSchemeParams.eventID, mTriggeredCollectionSchemeData->eventID ); - ASSERT_EQ( collectionSchemeParams.triggerTime, mTriggeredCollectionSchemeData->triggerTime ); - ASSERT_EQ( collectionSchemeParams.compression, mTriggeredCollectionSchemeData->metadata.compress ); - ASSERT_EQ( collectionSchemeParams.persist, mTriggeredCollectionSchemeData->metadata.persist ); + Schemas::VehicleDataMsg::VehicleData vehicleData; + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + ASSERT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_EQ( vehicleData.can_frames_size(), 0 ); + ASSERT_FALSE( vehicleData.has_dtc_data() ); + + ASSERT_EQ( vehicleData.captured_signals()[0].signal_id(), signal1.signalID ); + ASSERT_EQ( vehicleData.captured_signals()[0].double_value(), signal1.value.value.doubleVal ); + + // Ensure that there is no more data persisted + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); +} + +TEST_F( DataSenderManagerTest, PersistencyKeepFilesWhenRestarting ) +{ + std::shared_ptr mRawDataBufferManager; + mTriggeredCollectionSchemeData->metadata.persist = true; + auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); + mTriggeredCollectionSchemeData->signals.push_back( signal1 ); + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::TransmissionError ) ); + + processCollectedData( mTriggeredCollectionSchemeData ); + + // Now simulate a system restart + mDataSenderManager.reset(); + mPayloadManager.reset(); + mPersistency.reset(); + mPersistency = createCacheAndPersist(); + mPayloadManager = std::make_shared( mPersistency ); + std::unordered_map> dataSenders = { + { SenderDataType::TELEMETRY, + std::make_shared( + mMqttSender, mProtoWriter, mPayloadAdaptionConfigUncompressed, mPayloadAdaptionConfigCompressed ) } }; + mDataSenderManager = std::make_unique( dataSenders, mMqttSender, mPayloadManager ); + + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::Success ) ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); + auto sentBufferData = mMqttSender->getSentBufferData(); Schemas::VehicleDataMsg::VehicleData vehicleData; - std::string uncompressedData; - snappy::Uncompress( sentBufferData[0].data.c_str(), sentBufferData[0].data.size(), &uncompressedData ); - ASSERT_TRUE( vehicleData.ParseFromString( uncompressedData ) ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); ASSERT_EQ( vehicleData.captured_signals_size(), 1 ); ASSERT_EQ( vehicleData.can_frames_size(), 0 ); @@ -856,47 +1133,260 @@ TEST_F( DataSenderManagerTest, ProcessSingleSignalWithCompression ) ASSERT_EQ( vehicleData.captured_signals()[0].signal_id(), signal1.signalID ); ASSERT_EQ( vehicleData.captured_signals()[0].double_value(), signal1.value.value.doubleVal ); + + // Ensure that there is no more data persisted + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); } -TEST_F( DataSenderManagerTest, PersistencySingleFile ) +TEST_F( DataSenderManagerTest, PersistencyForTelemetryPersistAgainOnFailure ) { - Json::Value files( Json::arrayValue ); + mTriggeredCollectionSchemeData->metadata.persist = true; + auto signal1 = CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ); + mTriggeredCollectionSchemeData->signals.push_back( signal1 ); - files.append( Json::objectValue ); - files[0]["filename"] = "filename1"; - files[0]["compressionRequired"] = false; - files[0]["payloadSize"] = 1000; + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::NoConnection ) ); - EXPECT_CALL( *mPayloadManager, retrievePayloadMetadata( _ ) ) - .WillOnce( DoAll( SetArgReferee<0>( files ), Return( ErrorCode::SUCCESS ) ) ); + processCollectedData( mTriggeredCollectionSchemeData ); - EXPECT_CALL( *mMqttSender, sendFile( "filename1", 1000, _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::TransmissionError ) ); mDataSenderManager->checkAndSendRetrievedData(); + + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + + // Now the next attempt succeeds + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .WillOnce( InvokeArgument<2>( ConnectivityError::Success ) ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 1 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + Schemas::VehicleDataMsg::VehicleData vehicleData; + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + + ASSERT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + ASSERT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + ASSERT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + ASSERT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + ASSERT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_EQ( vehicleData.can_frames_size(), 0 ); + ASSERT_FALSE( vehicleData.has_dtc_data() ); + + ASSERT_EQ( vehicleData.captured_signals()[0].signal_id(), signal1.signalID ); + ASSERT_EQ( vehicleData.captured_signals()[0].double_value(), signal1.value.value.doubleVal ); + + // Ensure that there is no more data persisted + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + + mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); } -TEST_F( DataSenderManagerTest, PersistencyMultipleFiles ) +TEST_F( DataSenderManagerTest, PersistencyForTelemetryMultipleFiles ) { - Json::Value files( Json::arrayValue ); + mTriggeredCollectionSchemeData->metadata.persist = true; + + // The following should trigger 2 uploads, which will then be 2 files when persisting + int numberOfSignals; + size_t estimatedSize = 2 * ( mTriggeredCollectionSchemeData->metadata.collectionSchemeID.size() + + mTriggeredCollectionSchemeData->metadata.decoderID.size() + 12 ); + size_t maxSize = 2 * MAXIMUM_PAYLOAD_SIZE * mPayloadAdaptionConfigUncompressed.transmitThresholdStartPercent / 100; + for ( numberOfSignals = 0; ( estimatedSize + 20 ) < maxSize; numberOfSignals++ ) + { + auto signal = CollectedSignal( 1234, 700000 + numberOfSignals, numberOfSignals, SignalType::DOUBLE ); + mTriggeredCollectionSchemeData->signals.push_back( signal ); + estimatedSize += 20; + } - files.append( Json::objectValue ); - files[0]["filename"] = "filename1"; - files[0]["compressionRequired"] = false; - files[0]["payloadSize"] = 1000; + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .Times( 2 ) + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::NoConnection ) ); - files.append( Json::objectValue ); - files[1]["filename"] = "filename2"; - files[1]["compressionRequired"] = false; - files[1]["payloadSize"] = 3000; + processCollectedData( mTriggeredCollectionSchemeData ); + + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .Times( 2 ) + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); + + mDataSenderManager->checkAndSendRetrievedData(); - EXPECT_CALL( *mPayloadManager, retrievePayloadMetadata( _ ) ) - .WillOnce( DoAll( SetArgReferee<0>( files ), Return( ErrorCode::SUCCESS ) ) ); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 2 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + Schemas::VehicleDataMsg::VehicleData vehicleData; + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + + EXPECT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + EXPECT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + EXPECT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + EXPECT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + auto firstSize = vehicleData.captured_signals_size(); + ASSERT_TRUE( ( firstSize == ( numberOfSignals / 2 ) ) || ( firstSize == ( ( numberOfSignals / 2 ) + 1 ) ) ); + ASSERT_EQ( vehicleData.can_frames_size(), 0 ); + ASSERT_FALSE( vehicleData.has_dtc_data() ); + + for ( int i = 0; i < vehicleData.captured_signals_size(); i++ ) + { + EXPECT_EQ( vehicleData.captured_signals()[i].signal_id(), 1234 ); + EXPECT_EQ( vehicleData.captured_signals()[i].double_value(), i ); + } + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + + EXPECT_EQ( vehicleData.collection_event_id(), mTriggeredCollectionSchemeData->eventID ); + EXPECT_EQ( vehicleData.collection_event_time_ms_epoch(), mTriggeredCollectionSchemeData->triggerTime ); + EXPECT_EQ( vehicleData.campaign_sync_id(), mTriggeredCollectionSchemeData->metadata.collectionSchemeID ); + EXPECT_EQ( vehicleData.decoder_sync_id(), mTriggeredCollectionSchemeData->metadata.decoderID ); + + ASSERT_EQ( vehicleData.captured_signals_size(), numberOfSignals - firstSize ); + ASSERT_EQ( vehicleData.can_frames_size(), 0 ); + ASSERT_FALSE( vehicleData.has_dtc_data() ); + + for ( int i = 0; i < vehicleData.captured_signals_size(); i++ ) + { + EXPECT_EQ( vehicleData.captured_signals()[i].signal_id(), 1234 ); + EXPECT_EQ( vehicleData.captured_signals()[i].double_value(), i + firstSize ); + } - EXPECT_CALL( *mMqttSender, sendFile( "filename1", 1000, _ ) ) - .WillOnce( Return( ConnectivityError::WrongInputData ) ); - EXPECT_CALL( *mMqttSender, sendFile( "filename2", 3000, _ ) ).WillOnce( Return( ConnectivityError::Success ) ); + // Ensure that there is no more data persisted + mMqttSender->clearSentBufferData(); + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); mDataSenderManager->checkAndSendRetrievedData(); + + ASSERT_EQ( mMqttSender->getSentBufferData().size(), 0 ); +} + +TEST_F( DataSenderManagerTest, splitAndDecreaseThresholdWhenOverLimit ) +{ + // Change the maximum payload size, so that a payload will be built that doesn't fit, causing the transmit threshold + // to be iteratively decreased. The payloads are split, with the first few being dropped as splitting into quarters + // is still too big. + EXPECT_CALL( *mMqttSender, getMaxSendSize() ).Times( AnyNumber() ).WillRepeatedly( Return( 95 ) ); + + for ( auto i = 0; i < 100; i++ ) + { + mTriggeredCollectionSchemeData->signals.push_back( CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ) ); + } + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .Times( AtLeast( 12 ) ) + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); + + processCollectedData( mTriggeredCollectionSchemeData ); + + ASSERT_GE( mMqttSender->getSentBufferData().size(), 12 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + Schemas::VehicleDataMsg::VehicleData vehicleData; + + // Split into quarters fails. Threshold is decreased. + + // Split into quarters. Threshold is decreased. + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[2].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[3].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + // Split into halves. Threshold is decreased. + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[4].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[5].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + // Split into halves. Threshold is decreased. + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[6].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 1 ); + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[7].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + // Threshold now constant: + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[8].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[9].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[10].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[11].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 2 ); +} + +TEST_F( DataSenderManagerTest, increaseThresholdWhenBelowLimit ) +{ + // Change the maximum payload size, so the payload size is under the maximum limit, causing the transmit threshold + // to be iteratively increased: + EXPECT_CALL( *mMqttSender, getMaxSendSize() ) + .Times( AnyNumber() ) + .WillRepeatedly( Return( MAXIMUM_PAYLOAD_SIZE * 2 ) ); + + for ( auto i = 0; i < 200; i++ ) + { + mTriggeredCollectionSchemeData->signals.push_back( CollectedSignal( 1234, 789654, 40.5, SignalType::DOUBLE ) ); + } + + EXPECT_CALL( *mMqttSender, mockedSendBuffer( _, Gt( 0 ), _ ) ) + .Times( AtLeast( 9 ) ) + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); + + processCollectedData( mTriggeredCollectionSchemeData ); + + ASSERT_GE( mMqttSender->getSentBufferData().size(), 9 ); + auto sentBufferData = mMqttSender->getSentBufferData(); + + Schemas::VehicleDataMsg::VehicleData vehicleData; + // Increasing threshold: + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[0].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 14 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[1].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 16 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[2].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 17 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[3].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 19 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[4].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 21 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[5].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 24 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[6].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 26 ); + + // Threshold now constant: + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[7].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 29 ); + + ASSERT_TRUE( vehicleData.ParseFromString( sentBufferData[8].data ) ); + EXPECT_EQ( vehicleData.captured_signals_size(), 29 ); } } // namespace IoTFleetWise diff --git a/test/unit/DataSenderManagerWorkerThreadTest.cpp b/test/unit/DataSenderManagerWorkerThreadTest.cpp index 695add3f..deaa3e76 100644 --- a/test/unit/DataSenderManagerWorkerThreadTest.cpp +++ b/test/unit/DataSenderManagerWorkerThreadTest.cpp @@ -2,12 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 #include "DataSenderManagerWorkerThread.h" -#include "CANInterfaceIDTranslator.h" #include "Clock.h" #include "ClockHandler.h" #include "CollectionInspectionAPITypes.h" #include "ConnectivityModuleMock.h" #include "DataSenderManagerMock.h" +#include "DataSenderTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "Testing.h" #include "TimeTypes.h" @@ -18,11 +19,6 @@ #include #include -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -#include "ICollectionSchemeList.h" -#include -#endif - namespace Aws { namespace IoTFleetWise @@ -49,10 +45,12 @@ class DataSenderManagerWorkerThreadTest : public ::testing::Test mTriggeredCollectionSchemeData->eventID = 579; mConnectivityModule = std::make_shared>(); - mDataSenderManager = std::make_shared>( canIDTranslator ); - mCollectedDataQueue = std::make_shared( 10000 ); + mDataSenderManager = std::make_shared>(); + mCollectedDataQueue = std::make_shared( 10000, "Collected Data" ); + std::vector> dataToSendQueues; + dataToSendQueues.emplace_back( mCollectedDataQueue ); mDataSenderManagerWorkerThread = std::make_unique( - mConnectivityModule, mDataSenderManager, 100, mCollectedDataQueue ); + mConnectivityModule, mDataSenderManager, 100, dataToSendQueues ); EXPECT_CALL( *mConnectivityModule, isAlive() ).WillRepeatedly( Return( true ) ); } @@ -69,9 +67,8 @@ class DataSenderManagerWorkerThreadTest : public ::testing::Test std::shared_ptr mTriggeredCollectionSchemeData; std::shared_ptr> mConnectivityModule; - CANInterfaceIDTranslator canIDTranslator; std::shared_ptr> mDataSenderManager; - std::shared_ptr mCollectedDataQueue; + std::shared_ptr mCollectedDataQueue; std::unique_ptr mDataSenderManagerWorkerThread; }; @@ -83,16 +80,11 @@ class DataSenderManagerWorkerThreadTestWithAllSignalTypes : public DataSenderMan INSTANTIATE_TEST_SUITE_P( MultipleSignals, DataSenderManagerWorkerThreadTestWithAllSignalTypes, allSignalTypes, - signalTypeToString ); + signalTypeParamInfoToString ); TEST_F( DataSenderManagerWorkerThreadTest, ProcessNoData ) { -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _, _ ) ) -#else - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _ ) ) -#endif - .Times( 0 ); + EXPECT_CALL( *mDataSenderManager, mockedProcessData( _ ) ).Times( 1 ); mDataSenderManagerWorkerThread->start(); mCollectedDataQueue->push( mTriggeredCollectionSchemeData ); @@ -103,12 +95,7 @@ TEST_F( DataSenderManagerWorkerThreadTest, ProcessNoData ) TEST_P( DataSenderManagerWorkerThreadTestWithAllSignalTypes, ProcessSingleTrigger ) { SignalType signalType = GetParam(); -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _, _ ) ) -#else - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _ ) ) -#endif - .Times( 1 ); + EXPECT_CALL( *mDataSenderManager, mockedProcessData( _ ) ).Times( 1 ); mDataSenderManagerWorkerThread->start(); auto signal1 = CollectedSignal( 1234, mTriggerTime - 10, 40.5, signalType ); @@ -117,10 +104,11 @@ TEST_P( DataSenderManagerWorkerThreadTestWithAllSignalTypes, ProcessSingleTrigge mCollectedDataQueue->push( mTriggeredCollectionSchemeData ); WAIT_ASSERT_EQ( mDataSenderManager->getProcessedData().size(), 1U ); + auto processedData = mDataSenderManager->getProcessedData(); ASSERT_TRUE( mDataSenderManagerWorkerThread->stop() ); - ASSERT_EQ( mDataSenderManager->getProcessedData()[0]->signals.size(), 1 ); + ASSERT_EQ( processedData[0]->signals.size(), 1 ); - auto processedSignal = mDataSenderManager->getProcessedData()[0]->signals[0]; + auto processedSignal = processedData[0]->signals[0]; ASSERT_EQ( processedSignal.signalID, 1234 ); ASSERT_EQ( processedSignal.receiveTime, mTriggerTime - 10 ); ASSERT_NO_FATAL_FAILURE( assertSignalValue( processedSignal.value, 40.5, signalType ) ); @@ -134,12 +122,7 @@ TEST_F( DataSenderManagerWorkerThreadTest, ProcessMultipleTriggers ) triggeredCollectionSchemeData2->triggerTime = mTriggerTime; triggeredCollectionSchemeData2->eventID = 590; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _, _ ) ) -#else - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _ ) ) -#endif - .Times( 2 ); + EXPECT_CALL( *mDataSenderManager, mockedProcessData( _ ) ).Times( 2 ); mDataSenderManagerWorkerThread->start(); @@ -154,75 +137,20 @@ TEST_F( DataSenderManagerWorkerThreadTest, ProcessMultipleTriggers ) WAIT_ASSERT_EQ( mDataSenderManager->getProcessedData().size(), 2U ); ASSERT_TRUE( mDataSenderManagerWorkerThread->stop() ); + auto processedData = mDataSenderManager->getProcessedData(); - ASSERT_EQ( mDataSenderManager->getProcessedData()[0]->signals.size(), 1 ); - auto processedSignal = mDataSenderManager->getProcessedData()[0]->signals[0]; + ASSERT_EQ( processedData[0]->signals.size(), 1 ); + auto processedSignal = processedData[0]->signals[0]; ASSERT_EQ( processedSignal.signalID, 1234 ); ASSERT_EQ( processedSignal.receiveTime, mTriggerTime - 10 ); ASSERT_EQ( processedSignal.value.value.doubleVal, 40.5 ); - ASSERT_EQ( mDataSenderManager->getProcessedData()[1]->signals.size(), 1 ); - processedSignal = mDataSenderManager->getProcessedData()[1]->signals[0]; + ASSERT_EQ( processedData[1]->signals.size(), 1 ); + processedSignal = processedData[1]->signals[0]; ASSERT_EQ( processedSignal.signalID, 5678 ); ASSERT_EQ( processedSignal.receiveTime, mTriggerTime ); ASSERT_EQ( processedSignal.value.value.doubleVal, 99.5 ); } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -TEST_F( DataSenderManagerWorkerThreadTest, ProcessSingleTriggerWithRawData ) -{ - auto uploadedCollectionSchemeData = std::make_shared(); - uploadedCollectionSchemeData->metadata.decoderID = mTriggeredCollectionSchemeData->metadata.decoderID; - uploadedCollectionSchemeData->metadata.collectionSchemeID = - mTriggeredCollectionSchemeData->metadata.collectionSchemeID; - uploadedCollectionSchemeData->triggerTime = mTriggeredCollectionSchemeData->triggerTime; - uploadedCollectionSchemeData->eventID = mTriggeredCollectionSchemeData->eventID; - uploadedCollectionSchemeData->uploadedS3Objects.push_back( - UploadedS3Object{ "uploaded/object/key1", UploadedS3ObjectDataFormat::Cdr } ); - - // There should be two calls to process the data. The first will be the raw data to be uploaded - // to S3 and the second will be the message containing the S3 upload metadata (sent via MQTT). - Sequence seq; - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _, _ ) ) - .InSequence( seq ) - .WillOnce( InvokeArgument<1>( uploadedCollectionSchemeData ) ); - EXPECT_CALL( *mDataSenderManager, mockedProcessCollectedData( _, _ ) ).Times( 1 ).InSequence( seq ); - - mDataSenderManagerWorkerThread->start(); - - auto signal1 = CollectedSignal( 1234, mTriggerTime, 888999, SignalType::RAW_DATA_BUFFER_HANDLE ); - mTriggeredCollectionSchemeData->signals.push_back( signal1 ); - - mCollectedDataQueue->push( mTriggeredCollectionSchemeData ); - - WAIT_ASSERT_EQ( mDataSenderManager->getProcessedData().size(), 2U ); - ASSERT_TRUE( mDataSenderManagerWorkerThread->stop() ); - - ASSERT_EQ( mDataSenderManager->getProcessedData()[0]->signals.size(), 1 ); - ASSERT_EQ( mDataSenderManager->getProcessedData()[0]->uploadedS3Objects.size(), 0 ); - auto processedSignal = mDataSenderManager->getProcessedData()[0]->signals[0]; - ASSERT_EQ( processedSignal.signalID, 1234 ); - ASSERT_EQ( processedSignal.receiveTime, mTriggerTime ); - ASSERT_EQ( processedSignal.value.value.uint32Val, 888999 ); - - ASSERT_EQ( mDataSenderManager->getProcessedData()[1]->signals.size(), 0 ); - ASSERT_EQ( mDataSenderManager->getProcessedData()[1]->uploadedS3Objects.size(), 1 ); - auto uploadedS3Object = mDataSenderManager->getProcessedData()[1]->uploadedS3Objects[0]; - ASSERT_EQ( uploadedS3Object.key, "uploaded/object/key1" ); - ASSERT_EQ( uploadedS3Object.dataFormat, UploadedS3ObjectDataFormat::Cdr ); -} - -TEST_F( DataSenderManagerWorkerThreadTest, UpdateActiveCollectionSchemes ) -{ - auto activeCollectionSchemes = std::make_shared(); - - mDataSenderManagerWorkerThread->start(); - mDataSenderManagerWorkerThread->onChangeCollectionSchemeList( activeCollectionSchemes ); - - WAIT_ASSERT_TRUE( mDataSenderManager->mActiveCollectionSchemes != nullptr ); - ASSERT_EQ( mDataSenderManager->mActiveCollectionSchemes.get(), activeCollectionSchemes.get() ); -} -#endif - } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/DataSenderProtoWriterTest.cpp b/test/unit/DataSenderProtoWriterTest.cpp index 254eba2c..fcc86644 100644 --- a/test/unit/DataSenderProtoWriterTest.cpp +++ b/test/unit/DataSenderProtoWriterTest.cpp @@ -8,13 +8,14 @@ #include "ClockHandler.h" #include "CollectionInspectionAPITypes.h" #include "OBDDataTypes.h" +#include "RawDataManager.h" #include "SignalTypes.h" #include "TimeTypes.h" #include "vehicle_data.pb.h" #include #include -#include #include +#include #include #include #include @@ -27,26 +28,14 @@ namespace IoTFleetWise class DataSenderProtoWriterTest : public ::testing::Test { -public: - std::string - convertProtoToHex( const std::string &proto ) - { - std::string hex; - for ( size_t i = 0U; i < proto.size(); i++ ) - { - char hexBuf[3]; - sprintf( hexBuf, "%02X", (unsigned char)proto[i] ); - hex += hexBuf; - } - return hex; - } }; // Test the edge to cloud payload fields in the proto TEST_F( DataSenderProtoWriterTest, TestVehicleData ) { CANInterfaceIDTranslator canIDTranslator; - DataSenderProtoWriter protoWriter( canIDTranslator ); + std::shared_ptr mRawDataBufferManager; + DataSenderProtoWriter protoWriter( canIDTranslator, mRawDataBufferManager ); std::shared_ptr triggeredCollectionSchemeDataPtr = std::make_shared(); triggeredCollectionSchemeDataPtr->metadata.persist = false; @@ -61,26 +50,75 @@ TEST_F( DataSenderProtoWriterTest, TestVehicleData ) uint32_t collectionEventID = std::rand(); protoWriter.setupVehicleData( triggeredCollectionSchemeDataPtr, collectionEventID ); - CollectedSignal collectedSignalMsg( - 120 /*signalId*/, testTriggerTime + 2000 /*receiveTime*/, 77.88 /*value*/, SignalType::DOUBLE ); - protoWriter.append( collectedSignalMsg ); - EXPECT_EQ( protoWriter.getVehicleDataMsgCount(), 1 ); + protoWriter.append( CollectedSignal( 121, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 123 ), // value + SignalType::UINT8 ) ); + protoWriter.append( CollectedSignal( 122, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 456 ), // value + SignalType::UINT16 ) ); + protoWriter.append( CollectedSignal( 123, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 789 ), // value + SignalType::UINT32 ) ); + protoWriter.append( CollectedSignal( 124, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 55 ), // value + SignalType::UINT64 ) ); + protoWriter.append( CollectedSignal( 125, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( -123 ), // value + SignalType::INT8 ) ); + protoWriter.append( CollectedSignal( 126, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( -456 ), // value + SignalType::INT16 ) ); + protoWriter.append( CollectedSignal( 127, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( -789 ), // value + SignalType::INT32 ) ); + protoWriter.append( CollectedSignal( 128, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( -55 ), // value + SignalType::INT64 ) ); + protoWriter.append( CollectedSignal( 129, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 11.22 ), // value + SignalType::FLOAT ) ); + protoWriter.append( CollectedSignal( 130, // signalId + testTriggerTime + 2000, // receiveTime, + 77.88, // value + SignalType::DOUBLE ) ); + protoWriter.append( CollectedSignal( 130, // signalId + testTriggerTime + 2000, // receiveTime, + true, // value + SignalType::BOOLEAN ) ); + // Not supported, so won't increase estimated size + protoWriter.append( CollectedSignal( 131, // signalId + testTriggerTime + 2000, // receiveTime, + 0, // value + SignalType::UNKNOWN ) ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + // Not supported, so won't increase estimated size + protoWriter.append( CollectedSignal( 132, // signalId + testTriggerTime + 2000, // receiveTime, + 0, // value + SignalType::COMPLEX_SIGNAL ) ); +#endif + EXPECT_EQ( protoWriter.getVehicleDataEstimatedSize(), 242 ); std::array data = { 1, 2, 3, 4, 5, 6, 7, 8 }; CollectedCanRawFrame canRawFrameMsg( 12 /*frameId*/, 1 /*nodeId*/, testTriggerTime + 1000 /*receiveTime*/, data, 8 /*sizeof data*/ ); protoWriter.append( canRawFrameMsg ); - EXPECT_EQ( protoWriter.getVehicleDataMsgCount(), 2 ); + EXPECT_EQ( protoWriter.getVehicleDataEstimatedSize(), 266 ); std::string out; EXPECT_TRUE( protoWriter.serializeVehicleData( &out ) ); Schemas::VehicleDataMsg::VehicleData vehicleDataTest{}; - const std::string testProto = out; - if ( !vehicleDataTest.ParseFromString( testProto ) ) - { - ASSERT_TRUE( false ); - } + ASSERT_TRUE( vehicleDataTest.ParseFromString( out ) ); /* Read and compare to written fields */ ASSERT_EQ( "123", vehicleDataTest.campaign_sync_id() ); @@ -93,7 +131,8 @@ TEST_F( DataSenderProtoWriterTest, TestVehicleData ) TEST_F( DataSenderProtoWriterTest, TestDTCData ) { CANInterfaceIDTranslator canIDTranslator; - DataSenderProtoWriter protoWriter( canIDTranslator ); + std::shared_ptr mRawDataBufferManager; + DataSenderProtoWriter protoWriter( canIDTranslator, mRawDataBufferManager ); std::shared_ptr triggeredCollectionSchemeDataPtr = std::make_shared(); @@ -118,21 +157,17 @@ TEST_F( DataSenderProtoWriterTest, TestDTCData ) protoWriter.setupDTCInfo( dtcInfo ); protoWriter.append( dtcInfo.mDTCCodes[0] ); - EXPECT_EQ( protoWriter.getVehicleDataMsgCount(), 1 ); + EXPECT_EQ( protoWriter.getVehicleDataEstimatedSize(), 37 ); protoWriter.append( dtcInfo.mDTCCodes[1] ); - EXPECT_EQ( protoWriter.getVehicleDataMsgCount(), 2 ); + EXPECT_EQ( protoWriter.getVehicleDataEstimatedSize(), 44 ); std::string out; EXPECT_TRUE( protoWriter.serializeVehicleData( &out ) ); Schemas::VehicleDataMsg::VehicleData vehicleDataTest{}; - const std::string testProto = out; - if ( !vehicleDataTest.ParseFromString( testProto ) ) - { - ASSERT_TRUE( false ); - } + ASSERT_TRUE( vehicleDataTest.ParseFromString( out ) ); /* Read and compare to written fields */ ASSERT_EQ( "123", vehicleDataTest.campaign_sync_id() ); @@ -146,5 +181,116 @@ TEST_F( DataSenderProtoWriterTest, TestDTCData ) ASSERT_EQ( "P0456", dtcData->active_dtc_codes( 1 ) ); } +TEST_F( DataSenderProtoWriterTest, splitAndMerge ) +{ + CANInterfaceIDTranslator canIDTranslator; + std::shared_ptr mRawDataBufferManager; + DataSenderProtoWriter protoWriter( canIDTranslator, mRawDataBufferManager ); + std::shared_ptr triggeredCollectionSchemeDataPtr = + std::make_shared(); + triggeredCollectionSchemeDataPtr->metadata.persist = false; + triggeredCollectionSchemeDataPtr->metadata.compress = false; + triggeredCollectionSchemeDataPtr->metadata.priority = 0; + triggeredCollectionSchemeDataPtr->metadata.collectionSchemeID = "123"; + triggeredCollectionSchemeDataPtr->metadata.decoderID = "456"; + // Set the trigger time to current time + auto testClock = ClockHandler::getClock(); + Timestamp testTriggerTime = testClock->systemTimeSinceEpochMs(); + triggeredCollectionSchemeDataPtr->triggerTime = testTriggerTime; + + uint32_t collectionEventID = std::rand(); + protoWriter.setupVehicleData( triggeredCollectionSchemeDataPtr, collectionEventID ); + protoWriter.append( CollectedSignal( 121, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 123 ), // value + SignalType::UINT8 ) ); + protoWriter.append( CollectedSignal( 122, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 456 ), // value + SignalType::UINT16 ) ); + protoWriter.append( CollectedSignal( 123, // signalId + testTriggerTime + 2000, // receiveTime, + static_cast( 789 ), // value + SignalType::UINT32 ) ); + + DTCInfo dtcInfo; + dtcInfo.mSID = SID::STORED_DTC; + dtcInfo.receiveTime = testTriggerTime + 2000; + dtcInfo.mDTCCodes = { "U0123", "P0456" }; + + protoWriter.setupDTCInfo( dtcInfo ); + + protoWriter.append( dtcInfo.mDTCCodes[0] ); + protoWriter.append( dtcInfo.mDTCCodes[1] ); + + std::array canData = { 1, 2, 3, 4, 5, 6, 7, 8 }; + protoWriter.append( + CollectedCanRawFrame{ 12 /*frameId*/, 1 /*nodeId*/, testTriggerTime + 1000, canData, 8 /*size*/ } ); + protoWriter.append( + CollectedCanRawFrame{ 13 /*frameId*/, 1 /*nodeId*/, testTriggerTime + 2000, canData, 8 /*size*/ } ); + protoWriter.append( + CollectedCanRawFrame{ 14 /*frameId*/, 1 /*nodeId*/, testTriggerTime + 3000, canData, 8 /*size*/ } ); + protoWriter.append( + CollectedCanRawFrame{ 15 /*frameId*/, 1 /*nodeId*/, testTriggerTime + 3000, canData, 8 /*size*/ } ); + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + protoWriter.append( UploadedS3Object{ "abc1", UploadedS3ObjectDataFormat::Cdr } ); + protoWriter.append( UploadedS3Object{ "abc2", UploadedS3ObjectDataFormat::Cdr } ); + protoWriter.append( UploadedS3Object{ "abc3", UploadedS3ObjectDataFormat::Cdr } ); +#endif + + Schemas::VehicleDataMsg::VehicleData data; + protoWriter.splitVehicleData( data ); + + std::string out; + EXPECT_TRUE( protoWriter.serializeVehicleData( &out ) ); + + Schemas::VehicleDataMsg::VehicleData vehicleDataTest{}; + + ASSERT_TRUE( vehicleDataTest.ParseFromString( out ) ); + + /* Read and compare to written fields */ + ASSERT_EQ( "123", vehicleDataTest.campaign_sync_id() ); + ASSERT_EQ( "456", vehicleDataTest.decoder_sync_id() ); + ASSERT_EQ( collectionEventID, vehicleDataTest.collection_event_id() ); + ASSERT_EQ( testTriggerTime, vehicleDataTest.collection_event_time_ms_epoch() ); + + ASSERT_EQ( vehicleDataTest.captured_signals().size(), 1 ); + EXPECT_EQ( vehicleDataTest.captured_signals()[0].signal_id(), 121 ); + ASSERT_EQ( vehicleDataTest.dtc_data().active_dtc_codes_size(), 1 ); + EXPECT_EQ( vehicleDataTest.dtc_data().active_dtc_codes()[0], "U0123" ); + ASSERT_EQ( vehicleDataTest.can_frames().size(), 2 ); + EXPECT_EQ( vehicleDataTest.can_frames()[0].message_id(), 12 ); + EXPECT_EQ( vehicleDataTest.can_frames()[1].message_id(), 13 ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + ASSERT_EQ( vehicleDataTest.s3_objects().size(), 1 ); + EXPECT_EQ( vehicleDataTest.s3_objects()[0].key(), "abc1" ); +#endif + + protoWriter.mergeVehicleData( data ); + EXPECT_TRUE( protoWriter.serializeVehicleData( &out ) ); + ASSERT_TRUE( vehicleDataTest.ParseFromString( out ) ); + + /* Read and compare to written fields */ + ASSERT_EQ( "123", vehicleDataTest.campaign_sync_id() ); + ASSERT_EQ( "456", vehicleDataTest.decoder_sync_id() ); + ASSERT_EQ( collectionEventID, vehicleDataTest.collection_event_id() ); + ASSERT_EQ( testTriggerTime, vehicleDataTest.collection_event_time_ms_epoch() ); + + ASSERT_EQ( vehicleDataTest.captured_signals().size(), 2 ); + EXPECT_EQ( vehicleDataTest.captured_signals()[0].signal_id(), 122 ); + EXPECT_EQ( vehicleDataTest.captured_signals()[1].signal_id(), 123 ); + ASSERT_EQ( vehicleDataTest.dtc_data().active_dtc_codes_size(), 1 ); + EXPECT_EQ( vehicleDataTest.dtc_data().active_dtc_codes()[0], "P0456" ); + ASSERT_EQ( vehicleDataTest.can_frames().size(), 2 ); + EXPECT_EQ( vehicleDataTest.can_frames()[0].message_id(), 14 ); + EXPECT_EQ( vehicleDataTest.can_frames()[1].message_id(), 15 ); +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + ASSERT_EQ( vehicleDataTest.s3_objects().size(), 2 ); + EXPECT_EQ( vehicleDataTest.s3_objects()[0].key(), "abc2" ); + EXPECT_EQ( vehicleDataTest.s3_objects()[1].key(), "abc3" ); +#endif +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/DecoderDictionaryExtractorTest.cpp b/test/unit/DecoderDictionaryExtractorTest.cpp index 3dbcda3d..79cf4d2e 100644 --- a/test/unit/DecoderDictionaryExtractorTest.cpp +++ b/test/unit/DecoderDictionaryExtractorTest.cpp @@ -2,12 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 #include "CANInterfaceIDTranslator.h" +#include "CheckinSender.h" #include "Clock.h" #include "ClockHandler.h" +#include "CollectionSchemeManagerMock.h" #include "CollectionSchemeManagerTest.h" // IWYU pragma: associated #include "OBDDataTypes.h" #include "Testing.h" +#include "TimeTypes.h" #include +#include #include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -89,15 +93,13 @@ namespace IoTFleetWise * CollectionScheme3 is interested in signal 25 at Node 20 * CollectionScheme1 and CollectionScheme2 will be enabled at beginning. Later on CollectionScheme3 will be enabled. */ -TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) +TEST( DecoderDictionaryExtractorTest, DecoderDictionaryExtractorTest ) { - - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; canIDTranslator.add( "10" ); canIDTranslator.add( "20" ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); ASSERT_NE( canIDTranslator.getChannelNumericID( "10" ), canIDTranslator.getChannelNumericID( "20" ) ); - test.init( 0, nullptr, canIDTranslator ); std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ TimePoint currTime = testClock->timeSinceEpoch(); @@ -122,7 +124,7 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) ICollectionScheme::Signals_t signalInfo1 = ICollectionScheme::Signals_t(); // Generate 8 signals to be decoded and collected - for ( int i = 0; i < 8; ++i ) + for ( int i = 1; i < 9; ++i ) { SignalCollectionInfo signal; signal.signalID = i; @@ -141,17 +143,17 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) // This is CAN Frame 0x110 at Node 10 decoding format. It's part of decoder manifest struct CANMessageFormat canMessageFormat0x110; - // Signal 8 will be part of CAN Frame 0x110 at Node 10 - SignalCollectionInfo signal8; - signal8.signalID = 8; - signalInfo1.emplace_back( signal8 ); + // Signal 9 will be part of CAN Frame 0x110 at Node 10 + SignalCollectionInfo signal9; + signal9.signalID = 9; + signalInfo1.emplace_back( signal9 ); // CAN Frame 0x110 will only be decoded, its raw frame will not be collected - signalToFrameAndNodeID[signal8.signalID] = { 0x110, "10" }; + signalToFrameAndNodeID[signal9.signalID] = { 0x110, "10" }; canMessageFormat0x110.mMessageID = 0x101; canMessageFormat0x110.mSizeInBytes = 8; canMessageFormat0x110.mIsMultiplexed = false; CANSignalFormat sigFormat; - sigFormat.mSignalID = 8; + sigFormat.mSignalID = 9; canMessageFormat0x110.mSignals = { sigFormat }; // This vector defines a list of raw CAN Frame CollectionScheme1 wants to collect @@ -256,7 +258,6 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) inValidSignal.signalID = 0x10000; signalInfo2.emplace_back( inValidSignal ); - // Two collectionSchemes. ICollectionSchemePtr collectionScheme1 = std::make_shared( "COLLECTIONSCHEME1", "DM1", startTime1, stopTime1, signalInfo1, canFrameInfo1 ); ICollectionSchemePtr collectionScheme2 = std::make_shared( @@ -300,9 +301,9 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) ASSERT_EQ( decoderMethod.collectType, CANMessageCollectType::RAW_AND_DECODE ); // It shall contains Eight signal decoding formats. ASSERT_EQ( decoderMethod.format.mSignals.size(), 8 ); - for ( int signalID = 0; signalID < 8; ++signalID ) + for ( int i = 0; i < 8; ++i ) { - ASSERT_EQ( signalID, decoderMethod.format.mSignals[signalID].mSignalID ); + ASSERT_EQ( i + 1, decoderMethod.format.mSignals[i].mSignalID ); } } // Although 0x101 exit in Decoder Manifest but no CollectionScheme is interested in 0x101, hence decoder dictionary @@ -315,14 +316,14 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) auto decoderMethod = decoderDictionary->canMessageDecoderMethod[firstChannelId][0x200]; ASSERT_EQ( decoderMethod.collectType, CANMessageCollectType::RAW ); } - // CAN Frame 0x110 at Node 10 shall only have Signal 8 decoded. + // CAN Frame 0x110 at Node 10 shall only have Signal 9 decoded. ASSERT_EQ( decoderDictionary->canMessageDecoderMethod[firstChannelId].count( 0x110 ), 1 ); if ( decoderDictionary->canMessageDecoderMethod[firstChannelId].count( 0x110 ) == 1 ) { auto decoderMethod = decoderDictionary->canMessageDecoderMethod[firstChannelId][0x110]; ASSERT_EQ( decoderMethod.collectType, CANMessageCollectType::DECODE ); ASSERT_EQ( decoderMethod.format.mSignals.size(), 1 ); - ASSERT_EQ( decoderMethod.format.mSignals[0].mSignalID, 8 ); + ASSERT_EQ( decoderMethod.format.mSignals[0].mSignalID, 9 ); } // CAN Frame 0x200 at Node 20 shall have raw bytes collected and signal decoded. It contains signal 10 and 17 ASSERT_EQ( decoderDictionary->canMessageDecoderMethod[secondChannelId].count( 0x200 ), 1 ); @@ -425,8 +426,7 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) // The following code validates that when we have first the OBD signals in the decoder manifest // and then the CAN signals, the extraction still functions. The above code starts processing // always the CAN Signals first as the first network type is CAN. - CollectionSchemeManagerTest test2( "DM2" ); - test2.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test2( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM2" ); std::vector list2; // Two collectionSchemes with 5 seconds expiry/ // Timing is a problem on the target, making sure that we have a 100 ms of buffer @@ -459,12 +459,11 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorTest ) * This test aims to test CollectionScheme Manager's Decoder Dictionary Extractor functionality * when only a raw can frame is provided and no signals */ -TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorNoSignalsTest ) +TEST( DecoderDictionaryExtractorTest, DecoderDictionaryExtractorNoSignalsTest ) { - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ TimePoint currTime = testClock->timeSinceEpoch(); @@ -517,13 +516,12 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorNoSignalsTest ) * This test aims to test CollectionScheme Manager's Dececoder Dictionary Extractor functionality * when first a raw can frame is collected and then from the same frame a signal */ -TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorFirstRawFrameThenSignal ) +TEST( DecoderDictionaryExtractorTest, DecoderDictionaryExtractorFirstRawFrameThenSignal ) { - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; canIDTranslator.add( "10" ); - test.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ TimePoint currTime = testClock->timeSinceEpoch(); @@ -610,12 +608,10 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryExtractorFirstRawFrameThenSi } #ifdef FWE_FEATURE_VISION_SYSTEM_DATA -TEST( CollectionSchemeManagerTest, DecoderDictionaryComplexDataExtractor ) +TEST( DecoderDictionaryExtractorTest, DecoderDictionaryComplexDataExtractor ) { - - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ @@ -690,9 +686,12 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryComplexDataExtractor ) test.setCollectionSchemeList( PL1 ); // Both collectionScheme1 and collectionScheme2 are expected to be enabled ASSERT_TRUE( test.updateMapsandTimeLine( currTime ) ); + + auto inspectionMatrixOutput = std::make_shared(); + test.matrixExtractor( inspectionMatrixOutput ); // Invoke Decoder Dictionary Extractor function std::map> decoderDictionaryMap; - test.decoderDictionaryExtractor( decoderDictionaryMap ); + test.decoderDictionaryExtractor( decoderDictionaryMap, inspectionMatrixOutput ); auto dict = decoderDictionaryMap.find( VehicleDataSourceProtocol::COMPLEX_DATA ); ASSERT_NE( dict, decoderDictionaryMap.end() ); @@ -727,14 +726,35 @@ TEST( CollectionSchemeManagerTest, DecoderDictionaryComplexDataExtractor ) ASSERT_EQ( decodedPrimitive.mPrimitiveType, SignalType::UINT64 ); ASSERT_EQ( decodedPrimitive.mScaling, 1.0 ); ASSERT_EQ( decodedPrimitive.mOffset, 0.0 ); + + ASSERT_EQ( inspectionMatrixOutput->conditions.size(), 1 ); + ASSERT_EQ( inspectionMatrixOutput->conditions.at( 0 ).signals.size(), 6 ); + + auto &signals = inspectionMatrixOutput->conditions.at( 0 ).signals; + ASSERT_EQ( signals.at( 0 ).signalID, signal1.signalID ); + ASSERT_EQ( signals.at( 0 ).signalType, SignalType::UINT64 ); + + ASSERT_EQ( signals.at( 1 ).signalID, signal2.signalID ); + ASSERT_EQ( signals.at( 1 ).signalType, SignalType::UINT64 ); + + ASSERT_EQ( signals.at( 2 ).signalID, signal6.signalID ); + ASSERT_EQ( signals.at( 2 ).signalType, SignalType::UINT64 ); + + ASSERT_EQ( signals.at( 3 ).signalID, signal4.signalID ); + ASSERT_EQ( signals.at( 3 ).signalType, SignalType::UNKNOWN ); + + ASSERT_EQ( signals.at( 4 ).signalID, signal3.signalID ); + ASSERT_EQ( signals.at( 4 ).signalType, SignalType::UNKNOWN ); + + ASSERT_EQ( signals.at( 5 ).signalID, signal5.signalID ); + ASSERT_EQ( signals.at( 5 ).signalType, SignalType::UINT64 ); } -TEST( CollectionSchemeManagerTest, DecoderDictionaryInvalidPartialSignalIDAndInvalidComplexType ) +TEST( DecoderDictionaryExtractorTest, DecoderDictionaryInvalidPartialSignalIDAndInvalidComplexType ) { - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); std::shared_ptr testClock = ClockHandler::getClock(); /* mock currTime, and 3 collectionSchemes */ diff --git a/test/unit/ExternalCANDataSourceTest.cpp b/test/unit/ExternalCANDataSourceTest.cpp index b2d76299..e1d41ec4 100644 --- a/test/unit/ExternalCANDataSourceTest.cpp +++ b/test/unit/ExternalCANDataSourceTest.cpp @@ -6,6 +6,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "TimeTypes.h" #include "VehicleDataSourceTypes.h" @@ -86,6 +87,7 @@ class ExternalCANDataSourceTest : public ::testing::Test sigFormat1.mSizeInBits = 30; sigFormat1.mOffset = 0.0; sigFormat1.mFactor = 1.0; + sigFormat1.mSignalType = SignalType::DOUBLE; CANSignalFormat sigFormat2; sigFormat2.mSignalID = 7; @@ -95,6 +97,7 @@ class ExternalCANDataSourceTest : public ::testing::Test sigFormat2.mSizeInBits = 31; sigFormat2.mOffset = 0.0; sigFormat2.mFactor = 1.0; + sigFormat2.mSignalType = SignalType::DOUBLE; decoderMethod.format.mSignals.push_back( sigFormat1 ); decoderMethod.format.mSignals.push_back( sigFormat2 ); @@ -115,25 +118,30 @@ class ExternalCANDataSourceTest : public ::testing::Test TEST_F( ExternalCANDataSourceTest, testNoDecoderDictionary ) { - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); - CANDataConsumer consumer{ signalBufferPtr }; + CANDataConsumer consumer{ signalBufferDistributor }; ExternalCANDataSource dataSource{ consumer }; CollectedDataFrame collectedDataFrame; sendTestMessage( dataSource, 0 ); - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } TEST_F( ExternalCANDataSourceTest, testValidDecoderDictionary ) { - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); - CANDataConsumer consumer{ signalBufferPtr }; + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + + CANDataConsumer consumer{ signalBufferDistributor }; ExternalCANDataSource dataSource{ consumer }; dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; sendTestMessage( dataSource, 0 ); - ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); @@ -153,28 +161,31 @@ TEST_F( ExternalCANDataSourceTest, testValidDecoderDictionary ) // Test message a different message ID and non-monotonic time is not received sendTestMessage( dataSource, 0, 0x456, 1 ); - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); // Test invalidation of decoder dictionary dataSource.onChangeOfActiveDictionary( nullptr, VehicleDataSourceProtocol::RAW_SOCKET ); sendTestMessage( dataSource, 0 ); - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); // Check it ignores dictionaries for other protocols dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::OBD ); sendTestMessage( dataSource, 0 ); - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } TEST_F( ExternalCANDataSourceTest, testCanFDSocketMode ) { - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); - CANDataConsumer consumer{ signalBufferPtr }; + CANDataConsumer consumer{ signalBufferDistributor }; ExternalCANDataSource dataSource{ consumer }; dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; sendTestFDMessage( dataSource, 0 ); - ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); @@ -193,19 +204,22 @@ TEST_F( ExternalCANDataSourceTest, testCanFDSocketMode ) { ASSERT_EQ( frame->data[i], i ); } - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } TEST_F( ExternalCANDataSourceTest, testExtractExtendedID ) { - auto signalBufferPtr = std::make_shared( 10 ); + auto signalBuffer = std::make_shared( 10, "Signal Buffer" ); + + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); - CANDataConsumer consumer{ signalBufferPtr }; + CANDataConsumer consumer{ signalBufferDistributor }; ExternalCANDataSource dataSource{ consumer }; dataSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; sendTestMessageExtendedID( dataSource, 0 ); - ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto signal = collectedDataFrame.mCollectedSignals[0]; ASSERT_EQ( signal.value.type, SignalType::DOUBLE ); ASSERT_EQ( signal.signalID, 1 ); @@ -224,7 +238,7 @@ TEST_F( ExternalCANDataSourceTest, testExtractExtendedID ) { ASSERT_EQ( frame->data[i], i ); } - ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } } // namespace IoTFleetWise diff --git a/test/unit/ExternalGpsSourceTest.cpp b/test/unit/ExternalGpsSourceTest.cpp index 092d5192..53376676 100644 --- a/test/unit/ExternalGpsSourceTest.cpp +++ b/test/unit/ExternalGpsSourceTest.cpp @@ -5,6 +5,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "VehicleDataSourceTypes.h" #include "WaitUntil.h" @@ -52,20 +53,22 @@ class ExternalGpsSourceTest : public ::testing::Test // Test if valid gps data TEST_F( ExternalGpsSourceTest, testDecoding ) // NOLINT { - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); - ExternalGpsSource gpsSource( signalBufferPtr ); + auto signalBuffer = std::make_shared( 100, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + ExternalGpsSource gpsSource( signalBufferDistributor ); ASSERT_FALSE( gpsSource.init( INVALID_CAN_SOURCE_NUMERIC_ID, 1, 0, 32 ) ); ASSERT_TRUE( gpsSource.init( 1, 1, 0, 32 ) ); gpsSource.start(); gpsSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); gpsSource.setLocation( 360, 360 ); // Invalid gpsSource.setLocation( 52.5761, 12.5761 ); - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto firstSignal = collectedDataFrame.mCollectedSignals[0]; auto secondSignal = collectedDataFrame.mCollectedSignals[1]; @@ -77,7 +80,7 @@ TEST_F( ExternalGpsSourceTest, testDecoding ) // NOLINT ASSERT_TRUE( gpsSource.init( 1, 1, 123, 456 ) ); // Invalid start bits gpsSource.setLocation( 52.5761, 12.5761 ); - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); ASSERT_TRUE( gpsSource.stop() ); } @@ -85,17 +88,19 @@ TEST_F( ExternalGpsSourceTest, testDecoding ) // NOLINT // Test longitude west TEST_F( ExternalGpsSourceTest, testWestNegativeLongitude ) // NOLINT { - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); - ExternalGpsSource gpsSource( signalBufferPtr ); + SignalBufferPtr signalBuffer = std::make_shared( 100, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + ExternalGpsSource gpsSource( signalBufferDistributor ); gpsSource.init( 1, 1, 0, 32 ); gpsSource.start(); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); gpsSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); gpsSource.setLocation( 52.5761, -12.5761 ); - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto firstSignal = collectedDataFrame.mCollectedSignals[0]; auto secondSignal = collectedDataFrame.mCollectedSignals[1]; ASSERT_EQ( firstSignal.signalID, 0x1234 ); diff --git a/test/unit/ISOTPOverCANProtocolTest.cpp b/test/unit/ISOTPOverCANProtocolTest.cpp index 3126efce..35ed7837 100644 --- a/test/unit/ISOTPOverCANProtocolTest.cpp +++ b/test/unit/ISOTPOverCANProtocolTest.cpp @@ -39,7 +39,7 @@ class ISOTPOverCANProtocolTest : public ::testing::Test { if ( !socketAvailable() ) { - GTEST_SKIP() << "Skipping test fixture due to unavailability of socket"; + GTEST_FAIL() << "Test failed due to unavailability of socket"; } } diff --git a/test/unit/IWaveGpsSourceTest.cpp b/test/unit/IWaveGpsSourceTest.cpp index 54729d0a..6d56fb88 100644 --- a/test/unit/IWaveGpsSourceTest.cpp +++ b/test/unit/IWaveGpsSourceTest.cpp @@ -5,6 +5,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "VehicleDataSourceTypes.h" #include "WaitUntil.h" @@ -68,7 +69,9 @@ class IWaveGpsSourceTest : public ::testing::Test // Test if valid gps data TEST_F( IWaveGpsSourceTest, testDecoding ) { - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); + auto signalBuffer = std::make_shared( 100, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); // Random data so checksum etc. will not be valid *nmeaFile << "$GPGSV,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,25*26\n" "GPGSV,27,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,23,24,25*26\\n\n" @@ -76,17 +79,17 @@ TEST_F( IWaveGpsSourceTest, testDecoding ) "$GPGGA,133120.00,5234.56789,N,01234.56789,E,1,08,0.6,123.4,M,56.7,M,,*89\n\n" "$GPVTG,29.30,T,31.32,M,33.34,N,35.36,K,A*37C\n\n\n"; nmeaFile->close(); - IWaveGpsSource gpsSource( signalBufferPtr ); + IWaveGpsSource gpsSource( signalBufferDistributor ); ASSERT_FALSE( gpsSource.init( filePath, INVALID_CAN_SOURCE_NUMERIC_ID, 1, 0, 32 ) ); ASSERT_TRUE( gpsSource.init( filePath, 1, 1, 0, 32 ) ); gpsSource.connect(); gpsSource.start(); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( signalBufferPtr->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); gpsSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto firstSignal = collectedDataFrame.mCollectedSignals[0]; auto secondSignal = collectedDataFrame.mCollectedSignals[1]; @@ -104,18 +107,21 @@ TEST_F( IWaveGpsSourceTest, testDecoding ) // Test longitude west TEST_F( IWaveGpsSourceTest, testWestNegativeLongitude ) { - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); + SignalBufferPtr signalBuffer = std::make_shared( 100, "Signal Buffer" ); + auto signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); + // instead of E for east now, W for West *nmeaFile << "$GPGGA,133120.00,5234.56789,N,01234.56789,W,1,08,0.6,123.4,M,56.7,M,,*89\n\n"; nmeaFile->close(); - IWaveGpsSource gpsSource( signalBufferPtr ); + IWaveGpsSource gpsSource( signalBufferDistributor ); gpsSource.init( filePath, 1, 1, 0, 32 ); gpsSource.connect(); gpsSource.start(); gpsSource.onChangeOfActiveDictionary( mDictionary, VehicleDataSourceProtocol::RAW_SOCKET ); CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( signalBufferPtr->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto firstSignal = collectedDataFrame.mCollectedSignals[0]; auto secondSignal = collectedDataFrame.mCollectedSignals[1]; diff --git a/test/unit/InspectionMatrixExtractorTest.cpp b/test/unit/InspectionMatrixExtractorTest.cpp index a4bf7b40..6044da36 100644 --- a/test/unit/InspectionMatrixExtractorTest.cpp +++ b/test/unit/InspectionMatrixExtractorTest.cpp @@ -2,10 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 #include "CANInterfaceIDTranslator.h" +#include "CheckinSender.h" +#include "CollectionSchemeManagerMock.h" #include "CollectionSchemeManagerTest.h" // IWYU pragma: associated #include +#include #include #include +#include #ifdef FWE_FEATURE_VISION_SYSTEM_DATA #include "RawDataBufferManagerSpy.h" @@ -22,41 +26,43 @@ using ::testing::_; using ::testing::NiceMock; /** @brief - * This test aims to test PM's functionality to invoke Inspection Engine on Inspection Matrix Update - * step1: - * Two Inspection Engine registered as listener - * step2: Invoke Inspection Matrix Update - * check Both two Inspection Engines has Inspection Matrix Update + * This test aims to test PM's functionality to invoke Inspection Engine on Inspection Matrix update + * Step1: Register two Inspection Engines are listeners + * Step2: Invoke Matrix Updater and check if two Inspection Engines receive Inspection Matrix update */ -TEST( CollectionSchemeManagerTest, InspectionMatrixUpdaterTest ) +TEST( InspectionMatrixExtractorTest, MatrixUpdaterTest ) { - auto testPtr = std::make_shared(); CANInterfaceIDTranslator canIDTranslator; - testPtr->init( 0, nullptr, canIDTranslator ); + auto testPtr = std::make_shared( + nullptr, canIDTranslator, std::make_shared( nullptr ) ); // Mock two Inspection Engine Mock - std::shared_ptr InspectionEnginePtr; + std::shared_ptr InspectionEnginePtr1; std::shared_ptr InspectionEnginePtr2; - InspectionEnginePtr.reset( new CollectionInspectionWorkerThreadMock() ); + + InspectionEnginePtr1.reset( new CollectionInspectionWorkerThreadMock() ); InspectionEnginePtr2.reset( new CollectionInspectionWorkerThreadMock() ); + // Register two Inspection Engines as listeners to Inspection Matrix update testPtr->subscribeToInspectionMatrixChange( std::bind( &CollectionInspectionWorkerThreadMock::onChangeInspectionMatrix, - InspectionEnginePtr.get(), + InspectionEnginePtr1.get(), std::placeholders::_1 ) ); testPtr->subscribeToInspectionMatrixChange( std::bind( &CollectionInspectionWorkerThreadMock::onChangeInspectionMatrix, InspectionEnginePtr2.get(), std::placeholders::_1 ) ); - // Clear updater flag. Note this flag only exist in mock class for testing purpose - InspectionEnginePtr->setUpdateFlag( false ); - InspectionEnginePtr2->setUpdateFlag( false ); - // Invoke the updater + // Clear Inspection Matrix update flag (this flag only exist in mock class for testing purpose) + InspectionEnginePtr1->setInspectionMatrixUpdateFlag( false ); + InspectionEnginePtr2->setInspectionMatrixUpdateFlag( false ); + + // Invoke Inspection Matrix updater testPtr->inspectionMatrixUpdater( std::make_shared() ); - // Check both two consumers has updater flag set - ASSERT_TRUE( InspectionEnginePtr->getUpdateFlag() ); - ASSERT_TRUE( InspectionEnginePtr2->getUpdateFlag() ); + + // Check if both two consumers set Inspection Matrix update flag + ASSERT_TRUE( InspectionEnginePtr1->getInspectionMatrixUpdateFlag() ); + ASSERT_TRUE( InspectionEnginePtr2->getInspectionMatrixUpdateFlag() ); } ExpressionNode * @@ -143,7 +149,7 @@ deleteTree( ExpressionNode *root ) } /** @brief * This test mock 3 collectionSchemes, each with a binary tree of 20 nodes. - * Calls function inspectionMatrixExtractor + * Calls function matrixExtractor * Exam the output: printout flatten trees as well as traverse each tree * using ConditionWithCollectedData.condition * @@ -167,7 +173,8 @@ TEST( CollectionSchemeManager, InspectionMatrixExtractorTreeTest ) list1.emplace_back( collectionScheme2 ); list1.emplace_back( collectionScheme3 ); - CollectionSchemeManagerTest test( "DM1" ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); IDecoderManifestPtr DM1 = std::make_shared( "DM1" ); ICollectionSchemeListPtr PL1 = std::make_shared( list1 ); test.setDecoderManifest( DM1 ); @@ -175,7 +182,7 @@ TEST( CollectionSchemeManager, InspectionMatrixExtractorTreeTest ) // All three polices are expected to be enabled ASSERT_TRUE( test.updateMapsandTimeLine( { 0, 0 } ) ); std::shared_ptr output = std::make_shared(); - test.inspectionMatrixExtractor( output ); + test.matrixExtractor( output ); /* exam output */ for ( uint32_t i = 0; i < output->expressionNodeStorage.size(); i++ ) @@ -215,10 +222,9 @@ TEST( CollectionSchemeManager, InspectionMatrixExtractorConditionDataTest ) std::make_shared( "COLLECTIONSCHEME1", "DM1", 0, 10, testSignals, testCANFrames ); std::vector list1; list1.emplace_back( collectionScheme ); - CollectionSchemeManagerTest test( "DM1" ); CANInterfaceIDTranslator canIDTranslator; canIDTranslator.add( "102" ); - test.init( 0, nullptr, canIDTranslator ); + CollectionSchemeManagerWrapper test( nullptr, canIDTranslator, std::make_shared( nullptr ), "DM1" ); IDecoderManifestPtr DM1 = std::make_shared( "DM1" ); ICollectionSchemeListPtr PL1 = std::make_shared( list1 ); test.setDecoderManifest( DM1 ); @@ -226,7 +232,7 @@ TEST( CollectionSchemeManager, InspectionMatrixExtractorConditionDataTest ) ASSERT_TRUE( test.updateMapsandTimeLine( { 0, 0 } ) ); std::shared_ptr output = std::make_shared(); - test.inspectionMatrixExtractor( output ); + test.matrixExtractor( output ); for ( auto conditionData : output->conditions ) { // Signals @@ -252,7 +258,7 @@ TEST( CollectionSchemeManager, InspectionMatrixExtractorConditionDataTest ) /** @brief * This test aims to test PM's functionality to create and update the RawBuffer Config on Inspection Matrix Update */ -TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterTest ) +TEST( InspectionMatrixExtractorTest, InspectionMatrixRawBufferConfigUpdaterTest ) { struct SignalCollectionInfo signal1; signal1.signalID = 0x20001; // in complex data range of decoder manifest mock; @@ -280,9 +286,9 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterTest ) auto rawDataBufferManager = std::make_shared( RawData::BufferManagerConfig::create().get() ); - CollectionSchemeManagerTest test( "DMBM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator, rawDataBufferManager ); + CollectionSchemeManagerWrapper test( + nullptr, canIDTranslator, std::make_shared( nullptr ), "DMBM1", rawDataBufferManager ); IDecoderManifestPtr DMBM1 = std::make_shared( "DMBM1" ); ICollectionSchemeListPtr PL1 = std::make_shared( list1 ); test.setDecoderManifest( DMBM1 ); @@ -293,15 +299,17 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterTest ) ASSERT_TRUE( test.updateMapsandTimeLine( { 0, 0 } ) ); std::shared_ptr output = std::make_shared(); - test.updateRawDataBufferConfig( nullptr ); + std::unordered_map updatedSignals; + test.updateRawDataBufferConfigComplexSignals( nullptr, updatedSignals ); // The Config should be updated and 3 Raw Data Buffer should be Allocated + rawDataBufferManager->updateConfig( updatedSignals ); ASSERT_EQ( rawDataBufferManager->getActiveBuffers(), 3 ); } /** @brief * This test aims to test PM's functionality when the updated fails for the RawBuffer Config on Inspection Matrix Update */ -TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterMemLowTest ) +TEST( InspectionMatrixExtractorTest, InspectionMatrixRawBufferConfigUpdaterMemLowTest ) { struct SignalCollectionInfo signal1; signal1.signalID = 0x20001; // in complex data range of decoder manifest mock; @@ -332,9 +340,9 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterMemLowT RawData::BufferManagerConfig::create( maxBytes, reservedBytesPerSignal, {}, {}, {}, overridesPerSignal ) .get() ); - CollectionSchemeManagerTest test( "DMBM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator, rawDataBufferManager ); + CollectionSchemeManagerWrapper test( + nullptr, canIDTranslator, std::make_shared( nullptr ), "DMBM1", rawDataBufferManager ); IDecoderManifestPtr DMBM1 = std::make_shared( "DMBM1" ); ICollectionSchemeListPtr PL1 = std::make_shared( list1 ); test.setDecoderManifest( DMBM1 ); @@ -345,12 +353,14 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterMemLowT ASSERT_TRUE( test.updateMapsandTimeLine( { 0, 0 } ) ); std::shared_ptr output = std::make_shared(); - test.updateRawDataBufferConfig( nullptr ); + std::unordered_map updatedSignals; + test.updateRawDataBufferConfigComplexSignals( nullptr, updatedSignals ); + rawDataBufferManager->updateConfig( updatedSignals ); // The Config should be updated and have only 2 Raw Data Buffer due to limited memory ASSERT_EQ( rawDataBufferManager->getActiveBuffers(), 2 ); } -TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterWithComplexDataDictionary ) +TEST( InspectionMatrixExtractorTest, InspectionMatrixRawBufferConfigUpdaterWithComplexDataDictionary ) { struct SignalCollectionInfo signal1; signal1.signalID = 0x20001; // in complex data range of decoder manifest mock; @@ -378,9 +388,9 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterWithCom auto rawDataBufferManager = std::make_shared>( RawData::BufferManagerConfig::create().get() ); - CollectionSchemeManagerTest test( "DMBM1" ); CANInterfaceIDTranslator canIDTranslator; - test.init( 0, nullptr, canIDTranslator, rawDataBufferManager ); + CollectionSchemeManagerWrapper test( + nullptr, canIDTranslator, std::make_shared( nullptr ), "DMBM1", rawDataBufferManager ); std::unordered_map> formatMap; std::unordered_map> signalToFrameAndNodeID; @@ -421,7 +431,8 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterWithCom updatedSignals = arg; return RawData::BufferErrorCode::SUCCESSFUL; } ); - test.updateRawDataBufferConfig( complexDataDictionary ); + // Verify that the list of updatedSignals isn't overwritten + test.updateRawDataBufferConfigComplexSignals( complexDataDictionary, updatedSignals ); ASSERT_EQ( updatedSignals.size(), 3 ); @@ -442,6 +453,8 @@ TEST( CollectionSchemeManagerTest, InspectionMatrixRawBufferConfigUpdaterWithCom ASSERT_EQ( updatedSignals[signal3.signalID].interfaceId, "" ); ASSERT_EQ( updatedSignals[signal3.signalID].messageId, "" ); + rawDataBufferManager->updateConfig( updatedSignals ); + // The Config should be updated and 3 Raw Data Buffer should be Allocated ASSERT_EQ( rawDataBufferManager->getActiveBuffers(), 3 ); } diff --git a/test/unit/IoTFleetWiseEngineTest.cpp b/test/unit/IoTFleetWiseEngineTest.cpp index 6f179d39..918f692c 100644 --- a/test/unit/IoTFleetWiseEngineTest.cpp +++ b/test/unit/IoTFleetWiseEngineTest.cpp @@ -6,8 +6,11 @@ #include "CANDataTypes.h" #include "CollectionInspectionAPITypes.h" #include "IoTFleetWiseConfig.h" +#include "QueueTypes.h" +#include "SignalTypes.h" #include "WaitUntil.h" #include +#include #include #include #include @@ -92,7 +95,7 @@ class IoTFleetWiseEngineTest : public ::testing::Test { if ( !socketAvailable() ) { - GTEST_SKIP() << "Skipping test fixture due to unavailability of socket"; + GTEST_FAIL() << "Test failed due to unavailability of socket"; } #ifdef FWE_FEATURE_IWAVE_GPS std::ofstream iWaveGpsFile( "/tmp/engineTestIWaveGPSfile.txt" ); @@ -127,7 +130,8 @@ class IoTFleetWiseEngineTest : public ::testing::Test TEST_F( IoTFleetWiseEngineTest, InitAndStartEngine ) { Json::Value config; - ASSERT_TRUE( IoTFleetWiseConfig::read( "static-config-ok.json", config ) ); + std::string configFilePath = "static-config-ok.json"; + ASSERT_TRUE( IoTFleetWiseConfig::read( configFilePath, config ) ); IoTFleetWiseEngine engine; std::string keyPem; @@ -140,7 +144,7 @@ TEST_F( IoTFleetWiseEngineTest, InitAndStartEngine ) dummyCertificateFile << certPem; dummyCertificateFile.close(); - ASSERT_TRUE( engine.connect( config ) ); + ASSERT_TRUE( engine.connect( config, boost::filesystem::absolute( configFilePath ).parent_path() ) ); ASSERT_TRUE( engine.start() ); ASSERT_TRUE( engine.isAlive() ); @@ -151,7 +155,8 @@ TEST_F( IoTFleetWiseEngineTest, InitAndStartEngine ) TEST_F( IoTFleetWiseEngineTest, InitAndStartEngineInlineCreds ) { Json::Value config; - ASSERT_TRUE( IoTFleetWiseConfig::read( "static-config-inline-creds.json", config ) ); + std::string configFilePath = "static-config-inline-creds.json"; + ASSERT_TRUE( IoTFleetWiseConfig::read( configFilePath, config ) ); IoTFleetWiseEngine engine; std::string keyPem; @@ -161,7 +166,7 @@ TEST_F( IoTFleetWiseEngineTest, InitAndStartEngineInlineCreds ) config["staticConfig"]["mqttConnection"]["privateKey"] = keyPem; config["staticConfig"]["mqttConnection"]["rootCA"] = certPem; - ASSERT_TRUE( engine.connect( config ) ); + ASSERT_TRUE( engine.connect( config, boost::filesystem::absolute( configFilePath ).parent_path() ) ); ASSERT_TRUE( engine.start() ); ASSERT_TRUE( engine.isAlive() ); @@ -172,9 +177,10 @@ TEST_F( IoTFleetWiseEngineTest, InitAndStartEngineInlineCreds ) TEST_F( IoTFleetWiseEngineTest, CheckPublishDataQueue ) { Json::Value config; - ASSERT_TRUE( IoTFleetWiseConfig::read( "static-config-ok.json", config ) ); + std::string configFilePath = "static-config-ok.json"; + ASSERT_TRUE( IoTFleetWiseConfig::read( configFilePath, config ) ); IoTFleetWiseEngine engine; - ASSERT_TRUE( engine.connect( config ) ); + ASSERT_TRUE( engine.connect( config, boost::filesystem::absolute( configFilePath ).parent_path() ) ); // Push to the publish data queue std::shared_ptr collectedDataPtr = std::make_shared(); @@ -182,11 +188,14 @@ TEST_F( IoTFleetWiseEngineTest, CheckPublishDataQueue ) collectedDataPtr->metadata.decoderID = "456"; collectedDataPtr->triggerTime = 800; { - CollectedSignal collectedSignalMsg1( 120 /*signalId*/, 800 /*receiveTime*/, 77.88 /*value*/ ); + CollectedSignal collectedSignalMsg1( + 120 /*signalId*/, 800 /*receiveTime*/, 77.88 /*value*/, SignalType::DOUBLE ); collectedDataPtr->signals.push_back( collectedSignalMsg1 ); - CollectedSignal collectedSignalMsg2( 10 /*signalId*/, 1000 /*receiveTime*/, 46.5 /*value*/ ); + CollectedSignal collectedSignalMsg2( + 10 /*signalId*/, 1000 /*receiveTime*/, 46.5 /*value*/, SignalType::DOUBLE ); collectedDataPtr->signals.push_back( collectedSignalMsg2 ); - CollectedSignal collectedSignalMsg3( 12 /*signalId*/, 1200 /*receiveTime*/, 98.9 /*value*/ ); + CollectedSignal collectedSignalMsg3( + 12 /*signalId*/, 1200 /*receiveTime*/, 98.9 /*value*/, SignalType::DOUBLE ); collectedDataPtr->signals.push_back( collectedSignalMsg3 ); } { @@ -216,7 +225,7 @@ TEST_F( IoTFleetWiseEngineTest, InitAndFailToStartCorruptConfig ) ASSERT_TRUE( IoTFleetWiseConfig::read( "static-config-corrupt.json", config ) ); IoTFleetWiseEngine engine; // Connect should fail as the Config file has a non complete Bus definition - ASSERT_FALSE( engine.connect( config ) ); + ASSERT_FALSE( engine.connect( config, "static-config-corrupt.json" ) ); } } // namespace IoTFleetWise diff --git a/test/unit/LoggingModuleTest.cpp b/test/unit/LoggingModuleTest.cpp index c57f0f62..2b0f2588 100644 --- a/test/unit/LoggingModuleTest.cpp +++ b/test/unit/LoggingModuleTest.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "LoggingModule.h" +#include "ConsoleLogger.h" #include #include @@ -19,5 +20,17 @@ TEST( LoggingModuleTest, GetErrnoString ) ASSERT_NE( getErrnoString(), "" ); } +TEST( LoggingModuleTest, stringToLogColorOption ) +{ + LogColorOption outLogColorOption; + EXPECT_TRUE( stringToLogColorOption( "Auto", outLogColorOption ) ); + EXPECT_EQ( outLogColorOption, LogColorOption::Auto ); + EXPECT_TRUE( stringToLogColorOption( "Yes", outLogColorOption ) ); + EXPECT_EQ( outLogColorOption, LogColorOption::Yes ); + EXPECT_TRUE( stringToLogColorOption( "No", outLogColorOption ) ); + EXPECT_EQ( outLogColorOption, LogColorOption::No ); + EXPECT_FALSE( stringToLogColorOption( "Invalid", outLogColorOption ) ); +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/OBDDataDecoderTest.cpp b/test/unit/OBDDataDecoderTest.cpp index 1a5a93b0..a5074d3d 100644 --- a/test/unit/OBDDataDecoderTest.cpp +++ b/test/unit/OBDDataDecoderTest.cpp @@ -63,7 +63,7 @@ class OBDDataDecoderTest : public ::testing::Test } void - assertSignalValue( const OBDSignal &obdSignal, double expectedSignalValue, SignalType expectedSignalType ) + assertSignalValue( const DecodedSignalValue &obdSignal, double expectedSignalValue, SignalType expectedSignalType ) { switch ( expectedSignalType ) { @@ -117,7 +117,10 @@ class OBDDataDecoderTestWithAllSignalTypes : public OBDDataDecoderTest, public t { }; -INSTANTIATE_TEST_SUITE_P( AllSignals, OBDDataDecoderTestWithAllSignalTypes, allSignalTypes, signalTypeToString ); +INSTANTIATE_TEST_SUITE_P( AllSignals, + OBDDataDecoderTestWithAllSignalTypes, + allSignalTypes, + signalTypeParamInfoToString ); class OBDDataDecoderTestWithSignedSignalTypes : public OBDDataDecoderTest, public testing::WithParamInterface @@ -127,7 +130,7 @@ class OBDDataDecoderTestWithSignedSignalTypes : public OBDDataDecoderTest, INSTANTIATE_TEST_SUITE_P( SignedSignals, OBDDataDecoderTestWithSignedSignalTypes, signedSignalTypes, - signalTypeToString ); + signalTypeParamInfoToString ); TEST_P( OBDDataDecoderTestWithAllSignalTypes, FullSingleByte ) { diff --git a/test/unit/OBDOverCANModuleTest.cpp b/test/unit/OBDOverCANModuleTest.cpp index da523360..82f77c59 100644 --- a/test/unit/OBDOverCANModuleTest.cpp +++ b/test/unit/OBDOverCANModuleTest.cpp @@ -12,6 +12,7 @@ #include "MessageTypes.h" #include "OBDDataTypes.h" #include "OBDDataTypesUnitTestOnly.h" +#include "QueueTypes.h" #include "SignalTypes.h" #include "Testing.h" #include "Thread.h" @@ -44,9 +45,11 @@ class OBDOverCANModuleTest : public ::testing::Test { if ( !socketAvailable() ) { - GTEST_SKIP() << "Skipping test fixture due to unavailability of socket"; + GTEST_FAIL() << "Test failed due to unavailability of socket"; } - signalBufferPtr = std::make_shared( 256 ); + signalBuffer = std::make_shared( 256, "Signal Buffer" ); + signalBufferDistributor = std::make_shared(); + signalBufferDistributor->registerQueue( signalBuffer ); } void TearDown() override @@ -255,7 +258,8 @@ class OBDOverCANModuleTest : public ::testing::Test } OBDOverCANModule obdModule; - std::shared_ptr signalBufferPtr; + SignalBufferPtr signalBuffer; + SignalBufferDistributorPtr signalBufferDistributor; std::vector ecus; }; @@ -270,7 +274,9 @@ TEST_F( OBDOverCANModuleTest, OBDOverCANModuleInitTestSuccess ) { constexpr uint32_t obdPIDRequestInterval = 2; // seconds constexpr uint32_t obdDTCRequestInterval = 2; // seconds - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); ASSERT_TRUE( obdModule.disconnect() ); } @@ -296,7 +302,8 @@ TEST_F( OBDOverCANModuleTest, OBDOverCANModuleAndDecoderManifestLifecycle ) engineECUOptions.mDestinationCANId = toUType( ECU_ID_MOCK::ENGINE_ECU_TX ); ASSERT_TRUE( engineECU.init( engineECUOptions ) ); ASSERT_TRUE( engineECU.connect() ); - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // No Requests should be seen on the bus as it hasn't received a valid decoder dictionary yet. DELAY_ASSERT_FALSE( engineECU.receivePDU( ecmRxPDUData ) ); @@ -309,7 +316,10 @@ class OBDOverCANModuleTestWithAllSignalTypes : public OBDOverCANModuleTest, { }; -INSTANTIATE_TEST_SUITE_P( MultipleSignals, OBDOverCANModuleTestWithAllSignalTypes, allSignalTypes, signalTypeToString ); +INSTANTIATE_TEST_SUITE_P( MultipleSignals, + OBDOverCANModuleTestWithAllSignalTypes, + allSignalTypes, + signalTypeParamInfoToString ); TEST_P( OBDOverCANModuleTestWithAllSignalTypes, RequestPIDFromNotExtendedIDECUTest ) { @@ -338,7 +348,8 @@ TEST_P( OBDOverCANModuleTestWithAllSignalTypes, RequestPIDFromNotExtendedIDECUTe // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary( signalType ); @@ -352,7 +363,7 @@ TEST_P( OBDOverCANModuleTestWithAllSignalTypes, RequestPIDFromNotExtendedIDECUTe { toUType( EmissionPIDs::VEHICLE_SPEED ), 35 } }; // Verify all PID Signals are correctly decoded CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_NO_FATAL_FAILURE( @@ -390,7 +401,8 @@ TEST_F( OBDOverCANModuleTest, RequestPartialPIDFromNotExtendedIDECUTest ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary(); @@ -416,18 +428,18 @@ TEST_F( OBDOverCANModuleTest, RequestPartialPIDFromNotExtendedIDECUTest ) { 0xC1, 0xAA } }; // Verify all PID Signals are correctly decoded CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } // Check last two PID signals - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } - ASSERT_FALSE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } // This test is to validate that OBDOverCANModule can update the PID request list when receiving new decoder manifest @@ -464,7 +476,8 @@ TEST_F( OBDOverCANModuleTest, DecoderDictionaryUpdatePIDsToCollectTest ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary(); @@ -492,13 +505,13 @@ TEST_F( OBDOverCANModuleTest, DecoderDictionaryUpdatePIDsToCollectTest ) CollectedDataFrame collectedDataFrame; CollectedSignal signal; // Check first three PID signals - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } // Check last two PID signals - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); @@ -519,24 +532,24 @@ TEST_F( OBDOverCANModuleTest, DecoderDictionaryUpdatePIDsToCollectTest ) { 0xC1, 0xAA } }; // Verify all PID Signals are correctly decoded // Check first three PID signals - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } // Check last two PID signals - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto &signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } - ASSERT_FALSE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); // Update Decoder Dictionary to collect no signals. // publish the new decoder dictionary to OBD module obdModule.onChangeOfActiveDictionary( nullptr, VehicleDataSourceProtocol::OBD ); // We shall not receive any PIDs as decoder dictionary is empty - DELAY_ASSERT_FALSE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); } TEST_F( OBDOverCANModuleTest, RequestEmissionPIDAndDTCFromExtendedIDECUTest ) @@ -567,7 +580,8 @@ TEST_F( OBDOverCANModuleTest, RequestEmissionPIDAndDTCFromExtendedIDECUTest ) constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 1; // Request DTC every 1 seconds - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary(); @@ -583,21 +597,21 @@ TEST_F( OBDOverCANModuleTest, RequestEmissionPIDAndDTCFromExtendedIDECUTest ) { 0xC1, 0xAA } }; // Verify produced PID signals are correctly decoded CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } // Verify produced DTC Buffer is correct. 4 codes for ECM , 0 codes for TCM - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto dtcInfo = collectedDataFrame.mActiveDTCs; ASSERT_NE( dtcInfo, nullptr ); ASSERT_EQ( dtcInfo->mDTCCodes.size(), 4 ); @@ -635,7 +649,8 @@ TEST_F( OBDOverCANModuleTest, RequestPIDAndDTCFromNonExtendedIDECUTest ) constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 1; // Request DTC every 1 seconds - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); auto decoderDictPtr = initDecoderDictionary(); // publish decoder dictionary to OBD module @@ -652,20 +667,20 @@ TEST_F( OBDOverCANModuleTest, RequestPIDAndDTCFromNonExtendedIDECUTest ) CollectedDataFrame collectedDataFrame; // Verify produced PID signals are correctly decoded - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); } // Verify produced DTC Buffer is correct. 4 codes for ECM , 0 codes for TCM - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); auto dtcInfo = collectedDataFrame.mActiveDTCs; ASSERT_NE( dtcInfo, nullptr ); @@ -703,7 +718,8 @@ TEST_F( OBDOverCANModuleTest, BroadcastRequestsStandardIDs ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, true ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, true ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary(); @@ -717,7 +733,7 @@ TEST_F( OBDOverCANModuleTest, BroadcastRequestsStandardIDs ) { toUType( EmissionPIDs::VEHICLE_SPEED ), 35 } }; // Verify all PID Signals are correctly decoded CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { @@ -752,7 +768,8 @@ TEST_F( OBDOverCANModuleTest, BroadcastRequestsExtendedIDs ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 1; // 1 second constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, true ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, true ) ); ASSERT_TRUE( obdModule.connect() ); // Create decoder dictionary auto decoderDictPtr = initDecoderDictionary(); @@ -766,7 +783,7 @@ TEST_F( OBDOverCANModuleTest, BroadcastRequestsExtendedIDs ) { toUType( EmissionPIDs::VEHICLE_SPEED ), 35 } }; // Verify all PID Signals are correctly decoded CollectedDataFrame collectedDataFrame; - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); @@ -778,7 +795,8 @@ TEST_F( OBDOverCANModuleTest, getExternalPIDsToRequest ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 0; // no PID request constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); ASSERT_EQ( obdModule.getExternalPIDsToRequest(), std::vector() ); // Create decoder dictionary @@ -809,7 +827,8 @@ TEST_F( OBDOverCANModuleTest, setExternalPIDResponse ) // Request PIDs every 2 seconds and no DTC request constexpr uint32_t obdPIDRequestInterval = 0; // no PID request constexpr uint32_t obdDTCRequestInterval = 0; // no DTC request - ASSERT_TRUE( obdModule.init( signalBufferPtr, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); + ASSERT_TRUE( + obdModule.init( signalBufferDistributor, "vcan0", obdPIDRequestInterval, obdDTCRequestInterval, false ) ); ASSERT_TRUE( obdModule.connect() ); // Check before decoder manifest arrives: obdModule.setExternalPIDResponse( static_cast( EmissionPIDs::ENGINE_LOAD ), @@ -819,7 +838,7 @@ TEST_F( OBDOverCANModuleTest, setExternalPIDResponse ) // publish decoder dictionary to OBD module obdModule.onChangeOfActiveDictionary( decoderDictPtr, VehicleDataSourceProtocol::OBD ); CollectedDataFrame collectedDataFrame; - DELAY_ASSERT_FALSE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + DELAY_ASSERT_FALSE( signalBuffer->pop( collectedDataFrame ) ); // Check unknown PID: obdModule.setExternalPIDResponse( 0xCF, { 0x41, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00 } ); // Check incorrect length: @@ -840,7 +859,7 @@ TEST_F( OBDOverCANModuleTest, setExternalPIDResponse ) { toUType( EmissionPIDs::ENGINE_COOLANT_TEMPERATURE ), 70 }, { toUType( EmissionPIDs::VEHICLE_SPEED ), 35 } }; // Verify all PID Signals are correctly decoded - WAIT_ASSERT_TRUE( obdModule.getSignalBufferPtr()->pop( collectedDataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( collectedDataFrame ) ); for ( auto signal : collectedDataFrame.mCollectedSignals ) { ASSERT_DOUBLE_EQ( signal.value.value.doubleVal, expectedPIDSignalValue[signal.signalID] ); diff --git a/test/unit/PayloadManagerTest.cpp b/test/unit/PayloadManagerTest.cpp index 14aa9be2..6705eac9 100644 --- a/test/unit/PayloadManagerTest.cpp +++ b/test/unit/PayloadManagerTest.cpp @@ -3,16 +3,14 @@ #include "PayloadManager.h" #include "CacheAndPersist.h" -#include "ISender.h" -#include -#include +#include "Testing.h" +#include #include -#include #include #include #include +#include #include -#include #include namespace Aws @@ -22,235 +20,131 @@ namespace IoTFleetWise TEST( PayloadManagerTest, TestNoConnectionDataPersistency ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->erase( DataType::PAYLOAD_METADATA ); - persistencyPtr->init(); - PayloadManager testSend( persistencyPtr ); - - std::string testData1 = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; - testData1 += - '\0'; // make sure compression can handle null characters and does not stop after the first null character - testData1 += "testproto"; - - const uint8_t *stringData1 = reinterpret_cast( testData1.data() ); - - CollectionSchemeParams collectionSchemeParams1; - collectionSchemeParams1.compression = false; - collectionSchemeParams1.eventID = 123456; - collectionSchemeParams1.triggerTime = 123456; - - CollectionSchemeParams collectionSchemeParams2; - collectionSchemeParams2.compression = false; - collectionSchemeParams2.eventID = 678910; - collectionSchemeParams2.triggerTime = 678910; - - ASSERT_EQ( testSend.storeData( stringData1, testData1.size(), collectionSchemeParams1 ), true ); - - std::string filename; - - Json::Value files; - ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); - ASSERT_EQ( files[0]["filename"], - filename + std::to_string( collectionSchemeParams1.eventID ) + "-" + - std::to_string( collectionSchemeParams1.triggerTime ) + ".bin" ); - ASSERT_EQ( files[0]["payloadSize"].asUInt(), testData1.size() ); - ASSERT_EQ( files[0]["compressionRequired"], false ); - - filename = files[0]["filename"].asString(); - std::vector payload1( files[0]["payloadSize"].asUInt() ); - ASSERT_EQ( testSend.retrievePayload( payload1.data(), files[0]["payloadSize"].asUInt(), filename ), - ErrorCode::SUCCESS ); - ASSERT_EQ( testData1.size(), payload1.size() ); - ASSERT_TRUE( std::equal( testData1.begin(), testData1.begin() + testData1.size(), payload1.begin() ) ); - - persistencyPtr->erase( DataType::PAYLOAD_METADATA ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto persistencyPtr = createCacheAndPersist(); + + PayloadManager testSend( persistencyPtr ); + + std::string testData1 = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; + testData1 += + '\0'; // make sure compression can handle null characters and does not stop after the first null character + testData1 += "testproto"; + + const uint8_t *stringData1 = reinterpret_cast( testData1.data() ); + + Json::Value metadata; + metadata["someOtherField"] = 13; + std::string filename = "12345_09876.bin"; + + ASSERT_TRUE( testSend.storeData( stringData1, testData1.size(), metadata, filename ) ); + + Json::Value files; + ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); + ASSERT_EQ( files[0]["someOtherField"].asUInt(), 13 ); + + std::vector payload1( testData1.size() ); + ASSERT_EQ( testSend.retrievePayload( payload1.data(), testData1.size(), filename ), ErrorCode::SUCCESS ); + ASSERT_EQ( testData1.size(), payload1.size() ); + ASSERT_EQ( std::string( payload1.begin(), payload1.end() ), testData1 ); + + persistencyPtr->erase( DataType::PAYLOAD_METADATA ); +} + +TEST( PayloadManagerTest, TestNoConnectionDataPersistencyWithStreambuf ) +{ + auto persistencyPtr = createCacheAndPersist(); + + PayloadManager testSend( persistencyPtr ); + + std::string testData = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; + testData += + '\0'; // make sure compression can handle null characters and does not stop after the first null character + testData += "testproto"; + + std::stringstream stream( testData ); + + Json::Value metadata; + metadata["someOtherField"] = 13; + std::string filename = "12345_09876.bin"; + + ASSERT_TRUE( testSend.storeData( *stream.rdbuf(), metadata, filename ) ); + + Json::Value files; + ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); + // For now stream support is partial, we don't add the saved files to the metadata yet + ASSERT_EQ( files.size(), 0 ); + + std::vector payload( testData.size() ); + ASSERT_EQ( testSend.retrievePayload( payload.data(), testData.size(), filename ), ErrorCode::SUCCESS ); + ASSERT_EQ( std::string( payload.begin(), payload.end() ), testData ); } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA -TEST( PayloadManagerTest, TestNoConnectionDataPersistencyWithS3Upload ) +TEST( PayloadManagerTest, TestNoCacheAndPersistModuleWithStreambuf ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->erase( DataType::PAYLOAD_METADATA ); - persistencyPtr->init(); - PayloadManager testSend( persistencyPtr ); - - std::string testData1 = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; - testData1 += - '\0'; // make sure compression can handle null characters and does not stop after the first null character - testData1 += "testproto"; - - std::string testData2 = "testdata2"; - - const uint8_t *stringData1 = reinterpret_cast( testData1.data() ); - const uint8_t *stringData2 = reinterpret_cast( testData2.data() ); - - CollectionSchemeParams collectionSchemeParams1; - collectionSchemeParams1.compression = false; - collectionSchemeParams1.eventID = 123456; - collectionSchemeParams1.triggerTime = 123456; - - CollectionSchemeParams collectionSchemeParams2; - collectionSchemeParams2.compression = false; - collectionSchemeParams2.eventID = 678910; - collectionSchemeParams2.triggerTime = 678910; - - S3UploadParams s3UploadParams; - s3UploadParams.objectName = std::to_string( collectionSchemeParams2.eventID ) + "-" + - std::to_string( collectionSchemeParams2.triggerTime ) + ".10n"; - s3UploadParams.bucketName = "testBucket"; - s3UploadParams.bucketOwner = "012345678901"; - s3UploadParams.region = "us-west-1"; - s3UploadParams.uploadID = "123456"; - s3UploadParams.multipartID = 1; - - ASSERT_EQ( testSend.storeData( stringData1, testData1.size(), collectionSchemeParams1 ), true ); - ASSERT_EQ( testSend.storeData( stringData2, testData2.size(), collectionSchemeParams2, s3UploadParams ), true ); - - std::string filename; - - Json::Value files; - ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); - ASSERT_EQ( files[0]["filename"], - filename + std::to_string( collectionSchemeParams1.eventID ) + "-" + - std::to_string( collectionSchemeParams1.triggerTime ) + ".bin" ); - ASSERT_EQ( files[0]["payloadSize"].asUInt(), testData1.size() ); - ASSERT_EQ( files[0]["compressionRequired"], false ); - - ASSERT_EQ( files[1]["filename"], filename + s3UploadParams.objectName ); - ASSERT_EQ( files[1]["payloadSize"].asUInt(), testData2.size() ); - ASSERT_EQ( files[1]["compressionRequired"], false ); - ASSERT_EQ( files[1]["s3UploadMetadata"]["bucketName"], s3UploadParams.bucketName ); - ASSERT_EQ( files[1]["s3UploadMetadata"]["bucketOwner"], s3UploadParams.bucketOwner ); - ASSERT_EQ( files[1]["s3UploadMetadata"]["region"], s3UploadParams.region ); - ASSERT_EQ( files[1]["s3UploadMetadata"]["uploadID"], s3UploadParams.uploadID ); - ASSERT_EQ( files[1]["s3UploadMetadata"]["partNumber"], s3UploadParams.multipartID ); - - filename = files[0]["filename"].asString(); - std::vector payload1( files[0]["payloadSize"].asUInt() ); - ASSERT_EQ( testSend.retrievePayload( payload1.data(), files[0]["payloadSize"].asUInt(), filename ), - ErrorCode::SUCCESS ); - ASSERT_EQ( testData1.size(), payload1.size() ); - ASSERT_TRUE( std::equal( testData1.begin(), testData1.begin() + testData1.size(), payload1.begin() ) ); - - filename = files[1]["filename"].asString(); - std::vector payload2( files[1]["payloadSize"].asUInt() ); - ASSERT_EQ( testSend.retrievePayload( payload2.data(), files[1]["payloadSize"].asUInt(), filename ), - ErrorCode::SUCCESS ); - ASSERT_EQ( testData2.size(), payload2.size() ); - ASSERT_TRUE( std::equal( testData2.begin(), testData2.begin() + testData2.size(), payload2.begin() ) ); - - persistencyPtr->erase( DataType::PAYLOAD_METADATA ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + PayloadManager testSend( nullptr ); + + std::string testData = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; + std::stringstream stream( testData ); + + Json::Value metadata; + metadata["someOtherField"] = 13; + std::string filename = "12345_09876.bin"; + + ASSERT_FALSE( testSend.storeData( *stream.rdbuf(), metadata, filename ) ); + ASSERT_EQ( testSend.retrievePayload( nullptr, 0, filename ), ErrorCode::INVALID_DATA ); + Json::Value files; + ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::INVALID_DATA ); } -#endif TEST( PayloadManagerTest, TestNoCacheAndPersistModule ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - PayloadManager testSend( nullptr ); - - std::string testData = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; - - const uint8_t *stringData = reinterpret_cast( testData.data() ); - - CollectionSchemeParams collectionSchemeParams; - collectionSchemeParams.compression = false; - collectionSchemeParams.eventID = 123456; - collectionSchemeParams.triggerTime = 123456; - - std::string filename; - ASSERT_EQ( testSend.storeData( stringData, testData.size(), collectionSchemeParams ), false ); - ASSERT_EQ( testSend.retrievePayload( nullptr, 0, filename ), ErrorCode::INVALID_DATA ); - Json::Value files; - ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::INVALID_DATA ); - } + PayloadManager testSend( nullptr ); + + std::string testData = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; + + const uint8_t *stringData = reinterpret_cast( testData.data() ); + + Json::Value metadata; + metadata["someOtherField"] = 13; + + std::string filename = "12345_09876.bin"; + ASSERT_FALSE( testSend.storeData( stringData, testData.size(), metadata, filename ) ); + ASSERT_EQ( testSend.retrievePayload( nullptr, 0, filename ), ErrorCode::INVALID_DATA ); + Json::Value files; + ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::INVALID_DATA ); } TEST( PayloadManagerTest, TestEmptyBuffer ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 131072 ); - persistencyPtr->erase( DataType::PAYLOAD_METADATA ); - persistencyPtr->init(); - PayloadManager testSend( persistencyPtr ); - - CollectionSchemeParams collectionSchemeParams1; - collectionSchemeParams1.compression = false; - collectionSchemeParams1.eventID = 123456; - collectionSchemeParams1.triggerTime = 123456; - - std::string filename; - ASSERT_EQ( testSend.storeData( nullptr, 0, collectionSchemeParams1 ), false ); - ASSERT_EQ( testSend.retrievePayload( nullptr, 0, filename ), ErrorCode::INVALID_DATA ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto persistencyPtr = createCacheAndPersist(); + PayloadManager testSend( persistencyPtr ); + + Json::Value metadata; + std::string filename = "12345_09876.bin"; + + ASSERT_FALSE( testSend.storeData( nullptr, 0, metadata, filename ) ); + ASSERT_EQ( testSend.retrievePayload( nullptr, 0, filename ), ErrorCode::INVALID_DATA ); } TEST( PayloadManagerTest, TestFailToAddMetadata ) { - char buffer[PATH_MAX]; - if ( getcwd( buffer, sizeof( buffer ) ) != NULL ) - { - int ret = std::system( "mkdir ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - - const std::shared_ptr persistencyPtr = - std::make_shared( std::string( buffer ) + "/Persistency", 120 ); - persistencyPtr->init(); - PayloadManager testSend( persistencyPtr ); - - std::string testData1 = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; - const uint8_t *stringData1 = reinterpret_cast( testData1.data() ); - - CollectionSchemeParams collectionSchemeParams1; - collectionSchemeParams1.compression = false; - collectionSchemeParams1.eventID = 123456; - collectionSchemeParams1.triggerTime = 123456; - ASSERT_EQ( testSend.storeData( stringData1, testData1.size(), collectionSchemeParams1 ), false ); - - std::string filename = std::to_string( collectionSchemeParams1.eventID ) + "-" + - std::to_string( collectionSchemeParams1.triggerTime ) + ".bin"; - - Json::Value files; - std::vector payload1( testData1.size() ); - ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); - ASSERT_EQ( testSend.retrievePayload( payload1.data(), payload1.size(), filename ), ErrorCode::EMPTY ); - ASSERT_EQ( persistencyPtr->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename ), 0 ); - - ret = std::system( "rm -rf ./Persistency" ); - ASSERT_FALSE( WIFEXITED( ret ) == 0 ); - } + auto persistencyPtr = std::make_shared( getTempDir().string(), 120 ); + + PayloadManager testSend( persistencyPtr ); + + std::string testData1 = "abcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qaabcdefjh!24$iklmnop!24$3@qabbbb"; + const uint8_t *stringData1 = reinterpret_cast( testData1.data() ); + + Json::Value metadata; + metadata["someOtherField"] = 13; + ASSERT_FALSE( testSend.storeData( stringData1, testData1.size(), metadata, "12345_09876.bin" ) ); + + std::string filename = "12345_09876.bin"; + + Json::Value files; + std::vector payload1( testData1.size() ); + ASSERT_EQ( testSend.retrievePayloadMetadata( files ), ErrorCode::SUCCESS ); + ASSERT_EQ( testSend.retrievePayload( payload1.data(), payload1.size(), filename ), ErrorCode::EMPTY ); + ASSERT_EQ( persistencyPtr->getSize( DataType::EDGE_TO_CLOUD_PAYLOAD, filename ), 0 ); } } // namespace IoTFleetWise diff --git a/test/unit/CheckinAndPersistencyTest.cpp b/test/unit/PersistencyTest.cpp similarity index 75% rename from test/unit/CheckinAndPersistencyTest.cpp rename to test/unit/PersistencyTest.cpp index 90302f52..e164595c 100644 --- a/test/unit/CheckinAndPersistencyTest.cpp +++ b/test/unit/PersistencyTest.cpp @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include "CANInterfaceIDTranslator.h" -#include "Clock.h" -#include "ClockHandler.h" +#include "CacheAndPersist.h" +#include "CheckinSender.h" #include "CollectionSchemeManagerMock.h" #include "CollectionSchemeManagerTest.h" // IWYU pragma: associated #include @@ -18,55 +18,15 @@ namespace IoTFleetWise { using ::testing::_; -using ::testing::NiceMock; using ::testing::Return; using ::testing::ReturnRef; -/** @brief - * This test validates the retry logic of sending checkin messages. - * This is a non-connected test case ( no underlying MQTT Connection is created). - * This test validates that the scheduling logic is applied cyclicly. - */ -TEST( CollectionSchemeManagerTest2, checkInScheduleLogicTest ) -{ - // prepare input - std::string strDecoderManifestID1 = "DM1"; - std::string strCollectionSchemeIDCollectionScheme1 = "COLLECTIONSCHEME1"; - std::string strCollectionSchemeIDCollectionScheme2 = "COLLECTIONSCHEME2"; - - // create empty collectionScheme map - std::shared_ptr collectionScheme1 = std::make_shared(); - std::shared_ptr collectionScheme2 = std::make_shared(); - std::map mapEmpty; - std::map mapEnable = { - { strCollectionSchemeIDCollectionScheme1, collectionScheme1 }, - { strCollectionSchemeIDCollectionScheme2, collectionScheme2 } }; - - // setup maps - NiceMock gmocktest( strDecoderManifestID1, mapEnable, mapEmpty ); - std::shared_ptr testClock = ClockHandler::getClock(); - TimePoint currTime = testClock->timeSinceEpoch(); - // create mTimeLine - std::priority_queue, std::greater> testTimeLine; - TimeData dataPair = { currTime, "Checkin" }; - testTimeLine.push( dataPair ); - // test code - CANInterfaceIDTranslator canIDTranslator; - gmocktest.init( 200, nullptr, canIDTranslator ); - gmocktest.setTimeLine( testTimeLine ); - gmocktest.checkTimeLine( currTime ); - // We should have popped one item from the TimeLine, but also scheduled another one for the next cycle. - TimeData topPair = gmocktest.getTimeLine().top(); - ASSERT_EQ( topPair.time.monotonicTimeMs, currTime.monotonicTimeMs + 200 ); - ASSERT_EQ( topPair.id, "Checkin" ); -} - /** @brief * This test validates the usage of the store API of the persistency module. */ -TEST( CollectionSchemeManagerTest2, storeTest ) +TEST( PersistencyTest, storeTest ) { - std::string strDecoderManifestID1 = "DM1"; + SyncID strDecoderManifestID1 = "DM1"; // create testPersistency std::shared_ptr testPersistency = std::make_shared(); // build decodermanifest testDM1 @@ -76,7 +36,9 @@ TEST( CollectionSchemeManagerTest2, storeTest ) std::vector dataDM = { '1', '2', '3', '4' }; std::vector dataEmpty; - NiceMock gmocktest( strDecoderManifestID1 ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), strDecoderManifestID1 ); EXPECT_CALL( *testPL, getData() ) .WillOnce( ReturnRef( dataEmpty ) ) .WillOnce( ReturnRef( dataPL ) ) @@ -117,16 +79,18 @@ TEST( CollectionSchemeManagerTest2, storeTest ) /** @brief * This test validates the usage of the retrieve API of the persistency module. */ -TEST( CollectionSchemeManagerTest2, retrieveTest ) +TEST( PersistencyTest, retrieveTest ) { - std::string strDecoderManifestID1 = "DM1"; + SyncID strDecoderManifestID1 = "DM1"; // create testPersistency std::shared_ptr testPersistency = std::make_shared(); // build decodermanifest testDM1 std::shared_ptr testDM = std::make_shared(); std::shared_ptr testPL = std::make_shared(); - NiceMock gmocktest( strDecoderManifestID1 ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper gmocktest( + nullptr, canIDTranslator, std::make_shared( nullptr ), strDecoderManifestID1 ); EXPECT_CALL( *testPersistency, getSize( DataType::COLLECTION_SCHEME_LIST, _ ) ) .WillOnce( Return( 0 ) ) .WillOnce( Return( 100 ) ) @@ -170,7 +134,7 @@ TEST( CollectionSchemeManagerTest2, retrieveTest ) /** @brief * This test validates the usage of the store/retrieve APIs combined of the persistency module. */ -TEST( CollectionSchemeManagerTest2, StoreAndRetrieve ) +TEST( PersistencyTest, StoreAndRetrieve ) { int ret = std::system( "mkdir ./testPersist" ); ASSERT_FALSE( WIFEXITED( ret ) == 0 ); @@ -186,8 +150,9 @@ TEST( CollectionSchemeManagerTest2, StoreAndRetrieve ) size_t sizePL = dataPL.length(); size_t sizeDM = dataDM.length(); - CollectionSchemeManagerTest testCollectionSchemeManager; - testCollectionSchemeManager.setCollectionSchemePersistency( testPersistency ); + CANInterfaceIDTranslator canIDTranslator; + CollectionSchemeManagerWrapper testCollectionSchemeManager( + testPersistency, canIDTranslator, std::make_shared( nullptr ) ); std::vector emptyCP; ICollectionSchemeListPtr storePL = std::make_shared( emptyCP ); storePL->copyData( reinterpret_cast( dataPL.c_str() ), sizePL ); diff --git a/test/unit/ROS2DataSourceTest.cpp b/test/unit/ROS2DataSourceTest.cpp index 8a85678c..7f5be171 100644 --- a/test/unit/ROS2DataSourceTest.cpp +++ b/test/unit/ROS2DataSourceTest.cpp @@ -4,6 +4,7 @@ #include "CollectionInspectionAPITypes.h" #include "IDecoderDictionary.h" #include "MessageTypes.h" +#include "QueueTypes.h" #include "RawDataManager.h" #include "SignalTypes.h" #include "TimeTypes.h" @@ -75,7 +76,7 @@ class RawBufferManagerMock : public RawData::BufferManager // NOLINTNEXTLINE MOCK_METHOD( RawData::BufferHandle, push, - ( uint8_t * data, size_t size, Timestamp receiveTimestamp, RawData::BufferTypeId typeId ), + ( const uint8_t *data, size_t size, Timestamp receiveTimestamp, RawData::BufferTypeId typeId ), ( override ) ); }; @@ -88,6 +89,7 @@ class ROS2DataSourceTest : public ::testing::Test rclcpp::multiThreadedExecutorMock = &multiThreadedExecutorMock; rclcpp::nodeMock = &nodeMock; rclcpp::typeSupportMock = &typeSupportMock; + signalBufferDistributor->registerQueue( signalBuffer ); } void @@ -348,7 +350,8 @@ class ROS2DataSourceTest : public ::testing::Test std::shared_ptr> rawBufferManagerMock = std::make_shared>(); const int MINIMUM_WAIT_TIME_ONE_CYCLE_MS = 300; - SignalBufferPtr signalBufferPtr = std::make_shared( 100 ); + SignalBufferPtr signalBuffer = std::make_shared( 100, "Signal Buffer" ); + SignalBufferDistributorPtr signalBufferDistributor = std::make_shared(); public: std::atomic subscribeCallsCounter{ 0 }; @@ -388,7 +391,7 @@ TEST_F( ROS2DataSourceTest, NodeTest ) "topictest", "typetest", []( std::shared_ptr ) {}, 100 ), false ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 50, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + InterfaceID( "interface1" ), 50, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; EXPECT_CALL( nodeMock, create_generic_subscription ).WillRepeatedly( Throw( std::exception() ) ); ASSERT_EQ( ros2Node.subscribe( @@ -404,9 +407,9 @@ TEST_F( ROS2DataSourceTest, NodeTest ) TEST_F( ROS2DataSourceTest, SpinOnlyWithAvailableDictionary ) { - ROS2DataSourceConfig config{ std::string( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; + ROS2DataSourceConfig config{ InterfaceID( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; EXPECT_CALL( multiThreadedExecutorMock, spin ).Times( 0 ); - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); // Make sure thread does not start spinning so sleep for some time std::this_thread::sleep_for( 2 * std::chrono::milliseconds( MINIMUM_WAIT_TIME_ONE_CYCLE_MS ) ); @@ -426,8 +429,8 @@ TEST_F( ROS2DataSourceTest, FailedSanityCheckWithCompareOptionWarn ) .Times( 1 ) .WillRepeatedly( Return( nullptr ) ); - ROS2DataSourceConfig config{ std::string( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + ROS2DataSourceConfig config{ InterfaceID( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["interface1"]["messageIdTopicTest:messageIdTypeTest"].mCollectRaw = true; @@ -446,8 +449,8 @@ TEST_F( ROS2DataSourceTest, FailedSanityCheckWithCompareOptionError ) .WillRepeatedly( Return( nullptr ) ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["interface1"]["messageIdTopicTest:messageIdTypeTest"].mCollectRaw = true; @@ -464,8 +467,8 @@ TEST_F( ROS2DataSourceTest, FailedSanityCheckWithCompareOptionIgnore ) EXPECT_CALL( nodeMock, create_generic_subscription ).Times( 1 ).WillRepeatedly( Return( subscription ) ); EXPECT_CALL( typeSupportMock, get_typesupport_library( std::string( "messageIdTypeTest" ), _ ) ).Times( 0 ); - ROS2DataSourceConfig config{ std::string( "interface1" ), 2, CompareToIntrospection::NO_CHECK, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + ROS2DataSourceConfig config{ InterfaceID( "interface1" ), 2, CompareToIntrospection::NO_CHECK, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["interface1"]["messageIdTopicTest:messageIdTypeTest"].mCollectRaw = true; @@ -483,8 +486,8 @@ TEST_F( ROS2DataSourceTest, MessageIdWithColon ) .Times( 1 ) .WillRepeatedly( Return( subscription ) ); - ROS2DataSourceConfig config{ std::string( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + ROS2DataSourceConfig config{ InterfaceID( "interface1" ), 2, CompareToIntrospection::WARN_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["interface1"]["messageIdTopicTest:messageIdTypeTest"].mCollectRaw = true; @@ -502,8 +505,8 @@ TEST_F( ROS2DataSourceTest, MessageIdWithoutTypeTryUntilTypeFound ) { auto subscription = std::make_shared(); - ROS2DataSourceConfig config{ std::string( "interface1" ), 2, CompareToIntrospection::NO_CHECK, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + ROS2DataSourceConfig config{ InterfaceID( "interface1" ), 2, CompareToIntrospection::NO_CHECK, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); dictionary->complexMessageDecoderMethod["interface1"]["messageIdTopicTest"].mCollectRaw = true; @@ -539,8 +542,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareDifferentStructMemberCounts EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -563,8 +566,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareUnknownComplexType ) EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -586,8 +589,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareNoStructInDecodingInformatio EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -610,8 +613,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareNullptr ) EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -633,8 +636,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareDifferentStringPrimitiveType EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -657,8 +660,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareDifferentStringLength ) EXPECT_CALL( nodeMock, create_generic_subscription( _, _, _, _, _ ) ).Times( 0 ); ROS2DataSourceConfig config{ - std::string( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr ); + InterfaceID( "interface1" ), 2, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor ); ros2DataSource.connect(); auto dictionary = std::make_shared(); fillDefaultMessageType(); @@ -679,8 +682,8 @@ TEST_F( ROS2DataSourceTest, FailIntrospectionCompareDifferentStringLength ) TEST_F( ROS2DataSourceTest, SuccessfullyDecodeComplexMessage ) { ROS2DataSourceConfig config{ - std::string( "interface1" ), 50, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; - ROS2DataSource ros2DataSource( config, signalBufferPtr, rawBufferManagerMock ); + InterfaceID( "interface1" ), 50, CompareToIntrospection::ERROR_AND_FAIL_ON_DIFFERENCE, 100 }; + ROS2DataSource ros2DataSource( config, signalBufferDistributor, rawBufferManagerMock ); ros2DataSource.connect(); auto dictionary = std::make_shared(); @@ -693,7 +696,7 @@ TEST_F( ROS2DataSourceTest, SuccessfullyDecodeComplexMessage ) std::vector> dataElements; EXPECT_CALL( *rawBufferManagerMock, push( _, _, _, _ ) ) .Times( AtLeast( 1 ) ) - .WillRepeatedly( ( [&dataElements]( uint8_t *data, + .WillRepeatedly( ( [&dataElements]( const uint8_t *data, size_t size, Timestamp receiveTimestamp, RawData::BufferTypeId typeId ) -> RawData::BufferHandle { @@ -713,7 +716,7 @@ TEST_F( ROS2DataSourceTest, SuccessfullyDecodeComplexMessage ) ASSERT_EQ( dataElements.back().first, 123 ); CollectedDataFrame dataFrame; - WAIT_ASSERT_TRUE( signalBufferPtr->pop( dataFrame ) ); + WAIT_ASSERT_TRUE( signalBuffer->pop( dataFrame ) ); auto signalGroup = dataFrame.mCollectedSignals; auto signal1 = signalGroup[0]; @@ -721,18 +724,22 @@ TEST_F( ROS2DataSourceTest, SuccessfullyDecodeComplexMessage ) auto signal2 = signalGroup[1]; ASSERT_EQ( signal2.signalID, 0x80001112 ); - ASSERT_EQ( signal2.getValue().value.doubleVal, static_cast( 5 ) ); + ASSERT_EQ( signal2.getType(), SignalType::INT32 ); + ASSERT_EQ( signal2.getValue().value.int32Val, 5 ); auto signal3 = signalGroup[2]; ASSERT_EQ( signal3.signalID, 0x80001113 ); - ASSERT_EQ( signal3.getValue().value.doubleVal, static_cast( 10 ) ); + ASSERT_EQ( signal3.getType(), SignalType::INT32 ); + ASSERT_EQ( signal3.getValue().value.int32Val, 10 ); auto signal4 = signalGroup[3]; ASSERT_EQ( signal4.signalID, 0x80001114 ); - ASSERT_EQ( signal4.getValue().value.doubleVal, static_cast( 'u' ) ); + ASSERT_EQ( signal4.getType(), SignalType::UINT8 ); + ASSERT_EQ( signal4.getValue().value.uint8Val, 'u' ); auto signal5 = signalGroup[4]; ASSERT_EQ( signal5.signalID, 123 ); // first the raw buffer handle + ASSERT_EQ( signal5.getType(), SignalType::COMPLEX_SIGNAL ); ASSERT_EQ( signal5.getValue().value.uint32Val, 7890 ); ros2DataSource.disconnect(); diff --git a/test/unit/RemoteProfilerTest.cpp b/test/unit/RemoteProfilerTest.cpp index 3ed61b17..369c5785 100644 --- a/test/unit/RemoteProfilerTest.cpp +++ b/test/unit/RemoteProfilerTest.cpp @@ -3,11 +3,11 @@ #include "RemoteProfiler.h" #include "IConnectionTypes.h" -#include "ISender.h" #include "LogLevel.h" #include "SenderMock.h" #include "WaitUntil.h" #include +#include #include #include #include @@ -22,6 +22,7 @@ namespace IoTFleetWise using ::testing::_; using ::testing::Gt; +using ::testing::InvokeArgument; using ::testing::Return; using ::testing::Sequence; using ::testing::StrictMock; @@ -41,7 +42,7 @@ TEST( RemoteProfilerTest, MetricsUpload ) { auto senderMock = std::make_shared>(); EXPECT_CALL( *senderMock, mockedSendBuffer( _, Gt( 0 ), _ ) ) - .WillRepeatedly( Return( ConnectivityError::Success ) ); + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); auto mockLogSender = std::make_shared(); RemoteProfiler profiler( senderMock, mockLogSender, 1000, 1000, LogLevel::Trace, "Test" ); diff --git a/test/unit/S3SenderTest.cpp b/test/unit/S3SenderTest.cpp index caba094f..cbb3d944 100644 --- a/test/unit/S3SenderTest.cpp +++ b/test/unit/S3SenderTest.cpp @@ -11,7 +11,6 @@ #include "TransferManagerWrapperMock.h" #include #include -#include #include #include #include @@ -30,6 +29,7 @@ using ::testing::_; using ::testing::Exactly; using ::testing::MockFunction; using ::testing::Return; +using ::testing::SaveArg; using ::testing::StrictMock; namespace @@ -67,9 +67,7 @@ class S3SenderTest : public ::testing::Test std::shared_ptr> transferManagerWrapperMock; Aws::Transfer::TransferManagerConfiguration transferManagerConfiguration; - std::function( const Aws::Client::ClientConfiguration &, - Aws::Transfer::TransferManagerConfiguration & )> - createTransferManagerWrapper; + CreateTransferManagerWrapper createTransferManagerWrapper; }; class S3SenderCanceledStatusTest : public S3SenderTest, @@ -84,8 +82,8 @@ INSTANTIATE_TEST_SUITE_P( AllCanceledStatuses, TEST_P( S3SenderCanceledStatusTest, AsyncStreamUploadInitiatedCallbackCanceled ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 5 * 1024 * 1024 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 5 * 1024 * 1024 }; + MockFunction )> resultCallback; auto transferHandle = std::make_shared( TEST_BUCKET_NAME, TEST_OBJECT_KEY ); EXPECT_CALL( *transferManagerWrapperMock, @@ -96,36 +94,34 @@ TEST_P( S3SenderCanceledStatusTest, AsyncStreamUploadInitiatedCallbackCanceled ) _, _ ) ) .WillOnce( Return( transferHandle ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); - - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); + + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); ASSERT_NE( transferManagerConfiguration.transferStatusUpdatedCallback, nullptr ); - EXPECT_CALL( resultCallback, Call( false ) ).Times( 1 ); + EXPECT_CALL( resultCallback, Call( ConnectivityError::TransmissionError, _ ) ).Times( 1 ); transferHandle->UpdateStatus( GetParam() ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); } TEST_F( S3SenderTest, SendEmptyStream ) { - S3Sender sender{ nullptr, nullptr, 0 }; - ASSERT_EQ( - sender.sendStream( nullptr, - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - nullptr ), - ConnectivityError::WrongInputData ); + S3Sender sender{ nullptr, 0 }; + MockFunction )> resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::WrongInputData, _ ) ).Times( 1 ); + sender.sendStream( nullptr, + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); } TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackFailedFirstAttempt ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 5 * 1024 * 1024 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 5 * 1024 * 1024 }; + MockFunction )> resultCallback; auto transferHandle = std::make_shared( TEST_BUCKET_NAME, TEST_OBJECT_KEY ); EXPECT_CALL( *transferManagerWrapperMock, @@ -136,27 +132,25 @@ TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackFailedFirstAttempt ) _, _ ) ) .WillOnce( Return( transferHandle ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); - - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); + + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); ASSERT_NE( transferManagerConfiguration.transferStatusUpdatedCallback, nullptr ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::NOT_STARTED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); EXPECT_CALL( *transferManagerWrapperMock, RetryUpload( ::testing::A &>(), _ ) ) .WillOnce( Return( transferHandle ) ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::FAILED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + EXPECT_CALL( resultCallback, Call( ConnectivityError::Success, _ ) ).Times( 1 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::NOT_STARTED ); transferHandle->Restart(); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); @@ -165,8 +159,8 @@ TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackFailedFirstAttempt ) TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackFailedAllAttempts ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 5 * 1024 * 1024 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 5 * 1024 * 1024 }; + MockFunction )> resultCallback; auto transferHandle = std::make_shared( TEST_BUCKET_NAME, TEST_OBJECT_KEY ); EXPECT_CALL( *transferManagerWrapperMock, @@ -177,49 +171,53 @@ TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackFailedAllAttempts ) _, _ ) ) .WillOnce( Return( transferHandle ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); - - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); + + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); ASSERT_NE( transferManagerConfiguration.transferStatusUpdatedCallback, nullptr ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::NOT_STARTED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); EXPECT_CALL( *transferManagerWrapperMock, RetryUpload( ::testing::A &>(), _ ) ) .WillOnce( Return( transferHandle ) ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::FAILED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( false ) ).Times( 1 ); + std::shared_ptr failedData; + EXPECT_CALL( resultCallback, Call( ConnectivityError::TransmissionError, _ ) ) + .WillOnce( SaveArg<1>( &failedData ) ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::NOT_STARTED ); transferHandle->Restart(); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::FAILED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); + + ASSERT_NE( failedData, nullptr ); + std::stringstream ss; + ss << failedData.get(); + ASSERT_EQ( ss.str(), "test" ); } TEST_F( S3SenderTest, NoCredentialsProviderForStreamUpload ) { - S3Sender sender{ nullptr, nullptr, 0 }; - - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - nullptr ), - ConnectivityError::NotConfigured ); + S3Sender sender{ nullptr, 0 }; + MockFunction )> resultCallback; + EXPECT_CALL( resultCallback, Call( ConnectivityError::NotConfigured, _ ) ).Times( 1 ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); } TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackSucceeded ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 0 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 0 }; + MockFunction )> resultCallback; auto transferHandle = std::make_shared( TEST_BUCKET_NAME, TEST_OBJECT_KEY ); EXPECT_CALL( *transferManagerWrapperMock, @@ -230,33 +228,31 @@ TEST_F( S3SenderTest, AsyncStreamUploadInitiatedCallbackSucceeded ) _, _ ) ) .WillOnce( Return( transferHandle ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); - - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - TEST_OBJECT_KEY, - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); + + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + TEST_OBJECT_KEY, + resultCallback.AsStdFunction() ); ASSERT_NE( transferManagerConfiguration.transferStatusUpdatedCallback, nullptr ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::NOT_STARTED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::IN_PROGRESS ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); - EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + EXPECT_CALL( resultCallback, Call( ConnectivityError::Success, _ ) ).Times( 1 ); transferHandle->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle ); } TEST_F( S3SenderTest, LimitNumberOfSimultaneousUploadsAndQueueTheRemaining ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 0 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 0 }; + MockFunction )> resultCallback; auto transferHandle1 = std::make_shared( TEST_BUCKET_NAME, "objectKey1" ); EXPECT_CALL( *transferManagerWrapperMock, @@ -267,28 +263,22 @@ TEST_F( S3SenderTest, LimitNumberOfSimultaneousUploadsAndQueueTheRemaining ) _, _ ) ) .WillOnce( Return( transferHandle1 ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); // Hand over multiple files at once to the sender - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey1", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey1", + resultCallback.AsStdFunction() ); // The other files shouldn't be passed to transfer manager until the ongoing upload finishes - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey2", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey3", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey2", + resultCallback.AsStdFunction() ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey3", + resultCallback.AsStdFunction() ); // When the first upload completes, then the next should be sent auto transferHandle2 = std::make_shared( TEST_BUCKET_NAME, "objectKey2" ); @@ -300,7 +290,7 @@ TEST_F( S3SenderTest, LimitNumberOfSimultaneousUploadsAndQueueTheRemaining ) _, _ ) ) .WillOnce( Return( transferHandle2 ) ); - EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + EXPECT_CALL( resultCallback, Call( ConnectivityError::Success, _ ) ).Times( 1 ); // Complete the first upload transferHandle1->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle1 ); @@ -315,7 +305,7 @@ TEST_F( S3SenderTest, LimitNumberOfSimultaneousUploadsAndQueueTheRemaining ) _, _ ) ) .WillOnce( Return( transferHandle3 ) ); - EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + EXPECT_CALL( resultCallback, Call( ConnectivityError::Success, _ ) ).Times( 1 ); // Complete the second upload transferHandle2->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle2 ); @@ -323,8 +313,7 @@ TEST_F( S3SenderTest, LimitNumberOfSimultaneousUploadsAndQueueTheRemaining ) TEST_F( S3SenderTest, SkipQueuedUploadWhoseDataIsNotAvailableAnymore ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 0 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 0 }; auto transferHandle1 = std::make_shared( TEST_BUCKET_NAME, "objectKey1" ); EXPECT_CALL( *transferManagerWrapperMock, @@ -335,30 +324,29 @@ TEST_F( S3SenderTest, SkipQueuedUploadWhoseDataIsNotAvailableAnymore ) _, _ ) ) .WillOnce( Return( transferHandle1 ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); // Hand over multiple files at once to the sender - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey1", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + MockFunction )> resultCallback1; + EXPECT_CALL( resultCallback1, Call( _, _ ) ).Times( 0 ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey1", + resultCallback1.AsStdFunction() ); // Second file will return a null streambuf (as if its data were deleted), so it should be // skipped. - ASSERT_EQ( - sender.sendStream( - std::move( std::make_unique( std::unique_ptr( nullptr ) ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey2", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey3", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + MockFunction )> resultCallback2; + EXPECT_CALL( resultCallback2, Call( _, _ ) ).Times( 0 ); + sender.sendStream( + std::move( std::make_unique( std::unique_ptr( nullptr ) ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey2", + resultCallback2.AsStdFunction() ); + MockFunction )> resultCallback3; + EXPECT_CALL( resultCallback3, Call( _, _ ) ).Times( 0 ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey3", + resultCallback3.AsStdFunction() ); // When the first upload completes, then the second should be skipped and the third should be sent auto transferHandle3 = std::make_shared( TEST_BUCKET_NAME, "objectKey3" ); @@ -370,16 +358,22 @@ TEST_F( S3SenderTest, SkipQueuedUploadWhoseDataIsNotAvailableAnymore ) _, _ ) ) .WillOnce( Return( transferHandle3 ) ); - EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); // Complete the first upload transferHandle1->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); + EXPECT_CALL( resultCallback1, Call( ConnectivityError::Success, _ ) ).Times( 1 ); + EXPECT_CALL( resultCallback2, Call( ConnectivityError::WrongInputData, _ ) ).Times( 1 ); transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle1 ); + + // Complete the third upload + transferHandle3->UpdateStatus( Aws::Transfer::TransferStatus::COMPLETED ); + EXPECT_CALL( resultCallback3, Call( ConnectivityError::Success, _ ) ).Times( 1 ); + transferManagerConfiguration.transferStatusUpdatedCallback( nullptr, transferHandle3 ); } TEST_F( S3SenderTest, CancelAllOngoingUploadsOnDisconnection ) { - S3Sender sender{ nullptr, createTransferManagerWrapper, 0 }; - MockFunction resultCallback; + S3Sender sender{ createTransferManagerWrapper, 0 }; + MockFunction )> resultCallback; auto transferHandle1 = std::make_shared( TEST_BUCKET_NAME, "objectKey1" ); EXPECT_CALL( *transferManagerWrapperMock, @@ -390,27 +384,21 @@ TEST_F( S3SenderTest, CancelAllOngoingUploadsOnDisconnection ) _, _ ) ) .WillOnce( Return( transferHandle1 ) ); - EXPECT_CALL( resultCallback, Call( _ ) ).Times( 0 ); + EXPECT_CALL( resultCallback, Call( _, _ ) ).Times( 0 ); // Hand over multiple files at once to the sender - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey1", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey2", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); - ASSERT_EQ( - sender.sendStream( std::move( std::make_unique( "test" ) ), - S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, - "objectKey3", - resultCallback.AsStdFunction() ), - ConnectivityError::Success ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey1", + resultCallback.AsStdFunction() ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey2", + resultCallback.AsStdFunction() ); + sender.sendStream( std::move( std::make_unique( "test" ) ), + S3UploadMetadata{ TEST_BUCKET_NAME, TEST_PREFIX, TEST_REGION, TEST_BUCKET_OWNER_ACCOUNT_ID }, + "objectKey3", + resultCallback.AsStdFunction() ); EXPECT_CALL( *transferManagerWrapperMock, CancelAll() ); EXPECT_CALL( *transferManagerWrapperMock, MockedWaitUntilAllFinished( _ ) ); diff --git a/test/unit/SchemaTest.cpp b/test/unit/SchemaTest.cpp index d3c88461..c70522ef 100644 --- a/test/unit/SchemaTest.cpp +++ b/test/unit/SchemaTest.cpp @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 #include "Schema.h" -#include "AwsIotChannel.h" #include "AwsIotConnectivityModule.h" +#include "AwsIotReceiver.h" +#include "AwsIotSender.h" #include "Clock.h" #include "ClockHandler.h" #include "CollectionSchemeIngestion.h" @@ -13,7 +14,6 @@ #include "ICollectionSchemeList.h" #include "IConnectionTypes.h" #include "IDecoderManifest.h" -#include "ISender.h" #include "MessageTypes.h" #include "MqttClientWrapper.h" #include "SenderMock.h" @@ -24,10 +24,13 @@ #include "collection_schemes.pb.h" #include "common_types.pb.h" #include "decoder_manifest.pb.h" +#include #include #include #include +#include #include +#include #include #include #include @@ -49,17 +52,19 @@ namespace IoTFleetWise using ::testing::_; using ::testing::Gt; +using ::testing::InvokeArgument; +using ::testing::MockFunction; using ::testing::Return; using ::testing::Sequence; using ::testing::StrictMock; static void -assertCheckin( const std::string &data, const std::vector &sampleDocList, Timestamp timeBeforeCheckin ) +assertCheckin( const std::string &data, const std::vector &sampleDocList, Timestamp timeBeforeCheckin ) { std::shared_ptr clock = ClockHandler::getClock(); // Create a multiset of ARNS documents we have in a checkin to compare against was was put in the checkin - std::multiset documentSet( sampleDocList.begin(), sampleDocList.end() ); + std::multiset documentSet( sampleDocList.begin(), sampleDocList.end() ); // Deserialize the protobuf Schemas::CheckinMsg::Checkin sentCheckin; @@ -95,15 +100,15 @@ class SchemaTest : public ::testing::Test std::shared_ptr nullMqttClient; - mChannelDecoderManifest = - std::make_shared( mAwsIotModule.get(), nullptr, nullMqttClient, "topic", false ); - mChannelCollectionSchemeList = - std::make_shared( mAwsIotModule.get(), nullptr, nullMqttClient, "topic", false ); + mReceiverDecoderManifest = std::make_shared( mAwsIotModule.get(), nullMqttClient, "topic" ); + mReceiverCollectionSchemeList = + std::make_shared( mAwsIotModule.get(), nullMqttClient, "topic" ); mCollectionSchemeIngestion = std::make_unique( - mChannelDecoderManifest, - mChannelCollectionSchemeList, - std::make_shared( mAwsIotModule.get(), nullptr, nullMqttClient, "topic", false ) ); + mReceiverDecoderManifest, + mReceiverCollectionSchemeList, + std::make_shared( + mAwsIotModule.get(), nullMqttClient, "topic", Aws::Crt::Mqtt5::QOS::AWS_MQTT5_QOS_AT_MOST_ONCE ) ); mCollectionSchemeIngestion->subscribeToDecoderManifestUpdate( [&]( const IDecoderManifestPtr &decoderManifest ) { @@ -116,7 +121,7 @@ class SchemaTest : public ::testing::Test } static void - sendMessageToChannel( std::shared_ptr channel, google::protobuf::MessageLite &protoMsg ) + sendMessageToReceiver( std::shared_ptr receiver, google::protobuf::MessageLite &protoMsg ) { std::string protoSerializedBuffer; ASSERT_TRUE( protoMsg.SerializeToString( &protoSerializedBuffer ) ); @@ -127,12 +132,12 @@ class SchemaTest : public ::testing::Test publishPacket->WithPayload( Aws::Crt::ByteCursorFromArray( reinterpret_cast( protoSerializedBuffer.data() ), protoSerializedBuffer.length() ) ); - channel->onDataReceived( eventData ); + receiver->onDataReceived( eventData ); } std::unique_ptr mAwsIotModule; - std::shared_ptr mChannelDecoderManifest; - std::shared_ptr mChannelCollectionSchemeList; + std::shared_ptr mReceiverDecoderManifest; + std::shared_ptr mReceiverCollectionSchemeList; std::unique_ptr mCollectionSchemeIngestion; IDecoderManifestPtr mReceivedDecoderManifest; @@ -148,28 +153,30 @@ TEST_F( SchemaTest, Checkins ) auto senderMock = std::make_shared>(); std::shared_ptr nullMqttClient; - Schema collectionSchemeIngestion( - std::make_shared( awsIotModule.get(), nullptr, nullMqttClient, "topic", false ), - std::make_shared( awsIotModule.get(), nullptr, nullMqttClient, "topic", false ), - senderMock ); + Schema collectionSchemeIngestion( std::make_shared( awsIotModule.get(), nullMqttClient, "topic" ), + std::make_shared( awsIotModule.get(), nullMqttClient, "topic" ), + senderMock ); std::shared_ptr clock = ClockHandler::getClock(); Timestamp timeBeforeCheckin = clock->systemTimeSinceEpochMs(); // Create list of Arns - std::vector sampleDocList; + std::vector sampleDocList; Sequence seq; EXPECT_CALL( *senderMock, mockedSendBuffer( _, Gt( 0 ), _ ) ) .Times( 3 ) .InSequence( seq ) - .WillOnce( Return( ConnectivityError::Success ) ); + .WillRepeatedly( InvokeArgument<2>( ConnectivityError::Success ) ); EXPECT_CALL( *senderMock, mockedSendBuffer( _, Gt( 0 ), _ ) ) .InSequence( seq ) - .WillOnce( Return( ConnectivityError::NoConnection ) ); + .WillOnce( InvokeArgument<2>( ConnectivityError::NoConnection ) ); + + MockFunction resultCallback; // Test an empty checkin - ASSERT_TRUE( collectionSchemeIngestion.sendCheckin( sampleDocList ) ); + EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + collectionSchemeIngestion.sendCheckin( sampleDocList, resultCallback.AsStdFunction() ); ASSERT_EQ( senderMock->getSentBufferData().size(), 1 ); ASSERT_NO_FATAL_FAILURE( assertCheckin( senderMock->getSentBufferData()[0].data, sampleDocList, timeBeforeCheckin ) ); @@ -181,14 +188,17 @@ TEST_F( SchemaTest, Checkins ) sampleDocList.emplace_back( "DocArn4" ); // Test the previous doc list - ASSERT_TRUE( collectionSchemeIngestion.sendCheckin( sampleDocList ) ); + EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + collectionSchemeIngestion.sendCheckin( sampleDocList, resultCallback.AsStdFunction() ); // Test with duplicates - this shouldn't occur but make sure it works anyways sampleDocList.emplace_back( "DocArn4" ); - ASSERT_TRUE( collectionSchemeIngestion.sendCheckin( sampleDocList ) ); + EXPECT_CALL( resultCallback, Call( true ) ).Times( 1 ); + collectionSchemeIngestion.sendCheckin( sampleDocList, resultCallback.AsStdFunction() ); // Second call should simulate a offboardconnectivity issue, the checkin message should fail to send. - ASSERT_FALSE( collectionSchemeIngestion.sendCheckin( sampleDocList ) ); + EXPECT_CALL( resultCallback, Call( false ) ).Times( 1 ); + collectionSchemeIngestion.sendCheckin( sampleDocList, resultCallback.AsStdFunction() ); ASSERT_EQ( senderMock->getSentBufferData().size(), 4 ); ASSERT_NO_FATAL_FAILURE( assertCheckin( senderMock->getSentBufferData()[3].data, sampleDocList, timeBeforeCheckin ) ); @@ -234,6 +244,7 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) protoCANSignalB->set_offset( 100 ); protoCANSignalB->set_factor( 10 ); protoCANSignalB->set_length( 8 ); + protoCANSignalB->set_primitive_type( Schemas::DecoderManifestMsg::PrimitiveType::BOOL ); Schemas::DecoderManifestMsg::CANSignal *protoCANSignalC = protoDM.add_can_signals(); @@ -258,6 +269,7 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) protoOBDPIDSignalA->set_byte_length( 1 ); protoOBDPIDSignalA->set_bit_right_shift( 2 ); protoOBDPIDSignalA->set_bit_mask_length( 2 ); + protoOBDPIDSignalA->set_primitive_type( Schemas::DecoderManifestMsg::PrimitiveType::INT16 ); Schemas::DecoderManifestMsg::OBDPIDSignal *protoOBDPIDSignalB = protoDM.add_obd_pid_signals(); protoOBDPIDSignalB->set_signal_id( 567 ); @@ -270,14 +282,15 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) protoOBDPIDSignalB->set_byte_length( 2 ); protoOBDPIDSignalB->set_bit_right_shift( 0 ); protoOBDPIDSignalB->set_bit_mask_length( 8 ); + protoOBDPIDSignalB->set_primitive_type( Schemas::DecoderManifestMsg::PrimitiveType::UINT32 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelDecoderManifest, protoDM ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverDecoderManifest, protoDM ) ); // This should be false because we just copied the data and it needs to be built first ASSERT_FALSE( mReceivedDecoderManifest->isReady() ); // Assert that we get an empty string when we call getID on an object that's not yet built - ASSERT_EQ( mReceivedDecoderManifest->getID(), std::string() ); + ASSERT_EQ( mReceivedDecoderManifest->getID(), SyncID() ); ASSERT_TRUE( mReceivedDecoderManifest->build() ); ASSERT_TRUE( mReceivedDecoderManifest->isReady() ); @@ -291,26 +304,29 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) // Search the CANMessageFormat signals to find the signal format that corresponds to a specific signal // Then make sure the data matches the proto DecoderManifest definition of that signal - int found = 0; - for ( auto &sigFormat : testCMF.mSignals ) - { - if ( sigFormat.mSignalID == protoCANSignalA->signal_id() ) - { - found++; - ASSERT_EQ( protoCANSignalA->interface_id(), - mReceivedDecoderManifest->getCANFrameAndInterfaceID( sigFormat.mSignalID ).second ); - ASSERT_EQ( protoCANSignalA->message_id(), - mReceivedDecoderManifest->getCANFrameAndInterfaceID( sigFormat.mSignalID ).first ); - ASSERT_EQ( protoCANSignalA->is_big_endian(), sigFormat.mIsBigEndian ); - ASSERT_EQ( protoCANSignalA->is_signed(), sigFormat.mIsSigned ); - ASSERT_EQ( protoCANSignalA->start_bit(), sigFormat.mFirstBitPosition ); - ASSERT_EQ( protoCANSignalA->offset(), sigFormat.mOffset ); - ASSERT_EQ( protoCANSignalA->factor(), sigFormat.mFactor ); - ASSERT_EQ( protoCANSignalA->length(), sigFormat.mSizeInBits ); - } - } - // Assert that the one signal was found - ASSERT_EQ( found, 1 ); + auto sigFormat = + std::find_if( testCMF.mSignals.begin(), testCMF.mSignals.end(), [&]( const CANSignalFormat &format ) { + return format.mSignalID == protoCANSignalA->signal_id(); + } ); + + ASSERT_NE( sigFormat, testCMF.mSignals.end() ); + ASSERT_EQ( protoCANSignalA->interface_id(), + mReceivedDecoderManifest->getCANFrameAndInterfaceID( sigFormat->mSignalID ).second ); + ASSERT_EQ( protoCANSignalA->message_id(), + mReceivedDecoderManifest->getCANFrameAndInterfaceID( sigFormat->mSignalID ).first ); + ASSERT_EQ( protoCANSignalA->is_big_endian(), sigFormat->mIsBigEndian ); + ASSERT_EQ( protoCANSignalA->is_signed(), sigFormat->mIsSigned ); + ASSERT_EQ( protoCANSignalA->start_bit(), sigFormat->mFirstBitPosition ); + ASSERT_EQ( protoCANSignalA->offset(), sigFormat->mOffset ); + ASSERT_EQ( protoCANSignalA->factor(), sigFormat->mFactor ); + ASSERT_EQ( protoCANSignalA->length(), sigFormat->mSizeInBits ); + ASSERT_EQ( sigFormat->mSignalType, SignalType::DOUBLE ); + + sigFormat = std::find_if( testCMF.mSignals.begin(), testCMF.mSignals.end(), [&]( const CANSignalFormat &format ) { + return format.mSignalID == protoCANSignalB->signal_id(); + } ); + ASSERT_NE( sigFormat, testCMF.mSignals.end() ); + ASSERT_EQ( sigFormat->mSignalType, SignalType::BOOLEAN ); // Make sure we get a pair of Invalid CAN and Node Ids, for an signal that the the decoder manifest doesn't have ASSERT_EQ( mReceivedDecoderManifest->getCANFrameAndInterfaceID( 9999999 ), @@ -329,6 +345,7 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) ASSERT_EQ( protoOBDPIDSignalA->byte_length(), obdPIDDecoderFormat.mByteLength ); ASSERT_EQ( protoOBDPIDSignalA->bit_right_shift(), obdPIDDecoderFormat.mBitRightShift ); ASSERT_EQ( protoOBDPIDSignalA->bit_mask_length(), obdPIDDecoderFormat.mBitMaskLength ); + ASSERT_EQ( obdPIDDecoderFormat.mSignalType, SignalType::INT16 ); obdPIDDecoderFormat = mReceivedDecoderManifest->getPIDSignalDecoderFormat( 567 ); ASSERT_EQ( protoOBDPIDSignalB->pid_response_length(), obdPIDDecoderFormat.mPidResponseLength ); @@ -340,6 +357,7 @@ TEST_F( SchemaTest, DecoderManifestIngestion ) ASSERT_EQ( protoOBDPIDSignalB->byte_length(), obdPIDDecoderFormat.mByteLength ); ASSERT_EQ( protoOBDPIDSignalB->bit_right_shift(), obdPIDDecoderFormat.mBitRightShift ); ASSERT_EQ( protoOBDPIDSignalB->bit_mask_length(), obdPIDDecoderFormat.mBitMaskLength ); + ASSERT_EQ( obdPIDDecoderFormat.mSignalType, SignalType::UINT32 ); // There's no signal ID 890, hence this function shall return an INVALID_PID_DECODER_FORMAT obdPIDDecoderFormat = mReceivedDecoderManifest->getPIDSignalDecoderFormat( 890 ); @@ -364,13 +382,13 @@ TEST_F( SchemaTest, SchemaInvalidDecoderManifestTest ) protoDM.set_sync_id( "arn:aws:iam::123456789012:user/Development/product_1234/*" ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelDecoderManifest, protoDM ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverDecoderManifest, protoDM ) ); // This should be false because we just copied the data and it needs to be built first ASSERT_FALSE( mReceivedDecoderManifest->isReady() ); // Assert that we get an empty string when we call getID on an unbuilt object - ASSERT_EQ( mReceivedDecoderManifest->getID(), std::string() ); + ASSERT_EQ( mReceivedDecoderManifest->getID(), SyncID() ); ASSERT_FALSE( mReceivedDecoderManifest->build() ); ASSERT_FALSE( mReceivedDecoderManifest->isReady() ); @@ -408,13 +426,13 @@ TEST_F( SchemaTest, CollectionSchemeBasic ) auto p3 = protoCollectionSchemesMsg.add_collection_schemes(); // Make a list of collectionScheme ARNs - std::vector collectionSchemeARNs = { "P1", "P2", "P3" }; + std::vector collectionSchemeARNs = { "P1", "P2", "P3" }; p1->set_campaign_sync_id( collectionSchemeARNs[0] ); p2->set_campaign_sync_id( collectionSchemeARNs[1] ); p3->set_campaign_sync_id( collectionSchemeARNs[2] ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); // Try to build - this should succeed because we have real data ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); @@ -428,14 +446,18 @@ TEST_F( SchemaTest, CollectionSchemeBasic ) TEST_F( SchemaTest, EmptyCollectionSchemeIngestion ) { // Now we have data to pack our DecoderManifestIngestion object with! - CollectionSchemeIngestion collectionSchemeTest; + CollectionSchemeIngestion collectionSchemeTest{ +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + }; // isReady should evaluate to False ASSERT_TRUE( collectionSchemeTest.isReady() == false ); // Confirm that Message Metadata is not ready as Build has not been called - ASSERT_FALSE( collectionSchemeTest.getCollectionSchemeID().compare( std::string() ) ); - ASSERT_FALSE( collectionSchemeTest.getDecoderManifestID().compare( std::string() ) ); + ASSERT_FALSE( collectionSchemeTest.getCollectionSchemeID().compare( SyncID() ) ); + ASSERT_FALSE( collectionSchemeTest.getDecoderManifestID().compare( SyncID() ) ); ASSERT_TRUE( collectionSchemeTest.getStartTime() == std::numeric_limits::max() ); ASSERT_TRUE( collectionSchemeTest.getExpiryTime() == std::numeric_limits::max() ); ASSERT_TRUE( collectionSchemeTest.getAfterDurationMs() == std::numeric_limits::max() ); @@ -510,7 +532,7 @@ TEST_F( SchemaTest, CollectionSchemeIngestionHeartBeat ) can2->set_sample_buffer_size( 10 ); can2->set_minimum_sample_period_ms( 1000 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 1 ); @@ -595,6 +617,9 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) // Build the AST Tree: //---------- + const SignalID SIGNAL_ID_1 = 19; + const SignalID SIGNAL_ID_2 = 17; + const SignalID SIGNAL_ID_3 = 3; auto *root = new Schemas::CommonTypesMsg::ConditionNode(); message->set_allocated_condition_tree( root ); @@ -646,7 +671,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) auto *left_left_left = new Schemas::CommonTypesMsg::ConditionNode(); left_leftOp->set_allocated_left_child( left_left_left ); - left_left_left->set_node_signal_id( 19 ); + left_left_left->set_node_signal_id( SIGNAL_ID_1 ); auto *left_left_right = new Schemas::CommonTypesMsg::ConditionNode(); left_leftOp->set_allocated_right_child( left_left_right ); @@ -687,13 +712,13 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) right_rightOp->set_allocated_right_child( right_right_right ); auto *right_right_rightOp = new Schemas::CommonTypesMsg::ConditionNode_NodeOperator(); right_right_right->set_allocated_node_operator( right_right_rightOp ); - right_right_rightOp->set_operator_( Schemas::CommonTypesMsg::ConditionNode_NodeOperator_Operator_ARITHMETIC_MINUS ); + right_right_rightOp->set_operator_( Schemas::CommonTypesMsg::ConditionNode_NodeOperator_Operator_COMPARE_EQUAL ); //---------- auto *left_right_left_left = new Schemas::CommonTypesMsg::ConditionNode(); left_right_leftOp->set_allocated_left_child( left_right_left_left ); - left_right_left_left->set_node_signal_id( 19 ); + left_right_left_left->set_node_signal_id( SIGNAL_ID_1 ); auto *left_right_left_right = new Schemas::CommonTypesMsg::ConditionNode(); left_right_leftOp->set_allocated_right_child( left_right_left_right ); @@ -701,7 +726,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) auto *left_right_right_left = new Schemas::CommonTypesMsg::ConditionNode(); left_right_rightOp->set_allocated_left_child( left_right_right_left ); - left_right_right_left->set_node_signal_id( 19 ); + left_right_right_left->set_node_signal_id( SIGNAL_ID_1 ); auto *left_right_right_right = new Schemas::CommonTypesMsg::ConditionNode(); left_right_rightOp->set_allocated_right_child( left_right_right_right ); @@ -709,11 +734,11 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) auto *right_left_left_left = new Schemas::CommonTypesMsg::ConditionNode(); right_left_leftOp->set_allocated_left_child( right_left_left_left ); - right_left_left_left->set_node_signal_id( 19 ); + right_left_left_left->set_node_signal_id( SIGNAL_ID_1 ); auto *right_left_right_left = new Schemas::CommonTypesMsg::ConditionNode(); right_left_rightOp->set_allocated_left_child( right_left_right_left ); - right_left_right_left->set_node_signal_id( 19 ); + right_left_right_left->set_node_signal_id( SIGNAL_ID_1 ); auto *right_left_right_right = new Schemas::CommonTypesMsg::ConditionNode(); right_left_rightOp->set_allocated_right_child( right_left_right_right ); @@ -721,7 +746,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) auto *right_right_left_left = new Schemas::CommonTypesMsg::ConditionNode(); right_right_leftOp->set_allocated_left_child( right_right_left_left ); - right_right_left_left->set_node_signal_id( 19 ); + right_right_left_left->set_node_signal_id( SIGNAL_ID_1 ); auto *right_right_left_right = new Schemas::CommonTypesMsg::ConditionNode(); right_right_leftOp->set_allocated_right_child( right_right_left_right ); @@ -729,7 +754,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) auto *right_right_right_left = new Schemas::CommonTypesMsg::ConditionNode(); right_right_rightOp->set_allocated_left_child( right_right_right_left ); - right_right_right_left->set_node_signal_id( 19 ); + right_right_right_left->set_node_signal_id( SIGNAL_ID_1 ); auto *right_right_right_right = new Schemas::CommonTypesMsg::ConditionNode(); right_right_rightOp->set_allocated_right_child( right_right_right_right ); @@ -745,21 +770,21 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) // Add 3 Signals Schemas::CollectionSchemesMsg::SignalInformation *signal1 = collectionSchemeTestMessage->add_signal_information(); - signal1->set_signal_id( 19 ); + signal1->set_signal_id( SIGNAL_ID_1 ); signal1->set_sample_buffer_size( 5 ); signal1->set_minimum_sample_period_ms( 500 ); signal1->set_fixed_window_period_ms( 600 ); signal1->set_condition_only_signal( true ); Schemas::CollectionSchemesMsg::SignalInformation *signal2 = collectionSchemeTestMessage->add_signal_information(); - signal2->set_signal_id( 17 ); + signal2->set_signal_id( SIGNAL_ID_2 ); signal2->set_sample_buffer_size( 10000 ); signal2->set_minimum_sample_period_ms( 1000 ); signal2->set_fixed_window_period_ms( 1000 ); signal2->set_condition_only_signal( false ); Schemas::CollectionSchemesMsg::SignalInformation *signal3 = collectionSchemeTestMessage->add_signal_information(); - signal3->set_signal_id( 3 ); + signal3->set_signal_id( SIGNAL_ID_3 ); signal3->set_sample_buffer_size( 1000 ); signal3->set_minimum_sample_period_ms( 100 ); signal3->set_fixed_window_period_ms( 100 ); @@ -781,7 +806,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) collectionSchemeTestMessage->set_allocated_s3_upload_metadata( s3_upload_metadata ); #endif - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 1 ); @@ -801,19 +826,19 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) ASSERT_TRUE( collectionSchemeTest->isTriggerOnlyOnRisingEdge() == false ); // Signals ASSERT_TRUE( collectionSchemeTest->getCollectSignals().size() == 3 ); - ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).signalID == 19 ); + ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).signalID == SIGNAL_ID_1 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).sampleBufferSize == 5 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).minimumSampleIntervalMs == 500 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).fixedWindowPeriod == 600 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 0 ).isConditionOnlySignal == true ); - ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).signalID == 17 ); + ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).signalID == SIGNAL_ID_2 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).sampleBufferSize == 10000 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).minimumSampleIntervalMs == 1000 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).fixedWindowPeriod == 1000 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 1 ).isConditionOnlySignal == false ); - ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 2 ).signalID == 3 ); + ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 2 ).signalID == SIGNAL_ID_3 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 2 ).sampleBufferSize == 1000 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 2 ).minimumSampleIntervalMs == 100 ); ASSERT_TRUE( collectionSchemeTest->getCollectSignals().at( 2 ).fixedWindowPeriod == 100 ); @@ -833,7 +858,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) ASSERT_TRUE( collectionSchemeTest->getMinimumPublishIntervalMs() == 650 ); // Verify the AST - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 52 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 26 ); //---------- ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).nodeType, ExpressionNodeType::OPERATOR_LOGICAL_AND ); @@ -854,7 +879,7 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) //---------- ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->left->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->left->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->left->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->left->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->left->right->floatingValue, 1 ); @@ -869,39 +894,39 @@ TEST_F( SchemaTest, SchemaCollectionEventBased ) ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->nodeType, ExpressionNodeType::OPERATOR_ARITHMETIC_MINUS ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->nodeType, - ExpressionNodeType::OPERATOR_ARITHMETIC_MINUS ); + ExpressionNodeType::OPERATOR_EQUAL ); //---------- ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->left->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->left->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->left->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->left->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->left->right->floatingValue, 1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->right->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->right->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->right->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->right->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).left->right->right->right->floatingValue, 1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->left->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->left->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->left->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->left->right, nullptr ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->right->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->right->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->right->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->right->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->left->right->right->floatingValue, 1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->left->right->floatingValue, 1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->left->nodeType, ExpressionNodeType::SIGNAL ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->left->signalID, 19 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->left->signalID, SIGNAL_ID_1 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->right->nodeType, ExpressionNodeType::FLOAT ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).right->right->right->right->floatingValue, 1 ); @@ -1007,13 +1032,13 @@ TEST_F( SchemaTest, SchemaCollectionWithComplexTypes ) message->set_allocated_condition_tree( root ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 1 ); auto collectionSchemeTest = mReceivedCollectionSchemeList->getCollectionSchemes().at( 0 ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 10 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 5 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).nodeType, ExpressionNodeType::OPERATOR_EQUAL ); // assume first node is top root node @@ -1039,6 +1064,55 @@ TEST_F( SchemaTest, SchemaCollectionWithComplexTypes ) ASSERT_EQ( leftGeneratedSignalID, rightRightGeneratedSignalId ); } +TEST_F( SchemaTest, SchemaCollectionWithSamePartialSignal ) +{ + Schemas::CollectionSchemesMsg::CollectionSchemes protoCollectionSchemesMsg; + + auto protoCollectionScheme1 = protoCollectionSchemesMsg.add_collection_schemes(); + protoCollectionScheme1->set_campaign_sync_id( "campaign1" ); + protoCollectionScheme1->set_decoder_manifest_sync_id( "dm1" ); + protoCollectionScheme1->set_start_time_ms_epoch( 0 ); + protoCollectionScheme1->set_expiry_time_ms_epoch( 9262144816000 ); + + Schemas::CollectionSchemesMsg::SignalInformation *signal1 = protoCollectionScheme1->add_signal_information(); + signal1->set_signal_id( 200008 ); + signal1->set_sample_buffer_size( 100 ); + signal1->set_minimum_sample_period_ms( 1000 ); + signal1->set_fixed_window_period_ms( 1000 ); + signal1->set_condition_only_signal( false ); + + auto *signalPath1 = new Schemas::CommonTypesMsg::SignalPath(); + signalPath1->add_signal_path( 34574325 ); + signalPath1->add_signal_path( 5 ); + signalPath1->add_signal_path( 0 ); + signalPath1->add_signal_path( 42 ); + + signal1->set_allocated_signal_path( signalPath1 ); + + // Add another campaign with exactly the same config + auto protoCollectionScheme2 = protoCollectionSchemesMsg.add_collection_schemes(); + protoCollectionScheme2->CopyFrom( *protoCollectionScheme1 ); + protoCollectionScheme2->set_campaign_sync_id( "campaign2" ); + + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); + + ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); + ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 2 ); + auto collectionScheme1 = mReceivedCollectionSchemeList->getCollectionSchemes().at( 0 ); + auto collectionScheme2 = mReceivedCollectionSchemeList->getCollectionSchemes().at( 1 ); + + ASSERT_EQ( collectionScheme1->getCollectSignals().size(), 1 ); + ASSERT_EQ( collectionScheme2->getCollectSignals().size(), 1 ); + + // check its an internal generated ID + auto signalId1 = collectionScheme1->getCollectSignals().at( 0 ).signalID; + auto signalId2 = collectionScheme2->getCollectSignals().at( 0 ).signalID; + ASSERT_NE( signalId1 & INTERNAL_SIGNAL_ID_BITMASK, 0 ); + ASSERT_NE( signalId2 & INTERNAL_SIGNAL_ID_BITMASK, 0 ); + // Internal IDs should be reused across collection schemes if they refer to the same partial signal + ASSERT_EQ( signalId1, signalId2 ); +} + TEST_F( SchemaTest, SchemaCollectionWithDifferentWayToSpecifySignalIDInExpression ) { Schemas::CollectionSchemesMsg::CollectionSchemes protoCollectionSchemesMsg; @@ -1117,13 +1191,13 @@ TEST_F( SchemaTest, SchemaCollectionWithDifferentWayToSpecifySignalIDInExpressio message->set_allocated_condition_tree( root ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 1 ); auto collectionSchemeTest = mReceivedCollectionSchemeList->getCollectionSchemes().at( 0 ); - ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 14 ); + ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().size(), 7 ); ASSERT_EQ( collectionSchemeTest->getAllExpressionNodes().at( 0 ).nodeType, ExpressionNodeType::OPERATOR_EQUAL ); // assume first node is top root node @@ -1190,7 +1264,7 @@ TEST_F( SchemaTest, CollectionSchemeComplexHeartbeat ) path1->add_signal_path( 5 ); signal3->set_allocated_signal_path( path1 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelCollectionSchemeList, protoCollectionSchemesMsg ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverCollectionSchemeList, protoCollectionSchemesMsg ) ); ASSERT_TRUE( mReceivedCollectionSchemeList->build() ); ASSERT_EQ( mReceivedCollectionSchemeList->getCollectionSchemes().size(), 1 ); @@ -1308,7 +1382,7 @@ TEST_F( SchemaTest, DecoderManifestIngestionComplexSignals ) protoComplexSignal->set_message_id( "/topic/for/ROS:/vehicle/msgs/test.msg" ); protoComplexSignal->set_root_type_id( 20 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelDecoderManifest, protoDM ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverDecoderManifest, protoDM ) ); ASSERT_TRUE( mReceivedDecoderManifest->getComplexSignalDecoderFormat( 123 ).mInterfaceId.empty() ); @@ -1411,7 +1485,7 @@ TEST_F( SchemaTest, DecoderManifestWrong ) protoComplexSignal2->set_message_id( "/topic/for/ROS:/vehicle/msgs/test2.msg" ); protoComplexSignal2->set_root_type_id( 10 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelDecoderManifest, protoDM ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverDecoderManifest, protoDM ) ); ASSERT_TRUE( mReceivedDecoderManifest->build() ); ASSERT_TRUE( mReceivedDecoderManifest->isReady() ); @@ -1477,7 +1551,7 @@ TEST_F( SchemaTest, DecoderManifestIngestionComplexStringAsArray ) protoComplexSignal->set_message_id( "/topic/for/ROS:/vehicle/msgs/test.msg" ); protoComplexSignal->set_root_type_id( 20 ); - ASSERT_NO_FATAL_FAILURE( sendMessageToChannel( mChannelDecoderManifest, protoDM ) ); + ASSERT_NO_FATAL_FAILURE( sendMessageToReceiver( mReceiverDecoderManifest, protoDM ) ); ASSERT_TRUE( mReceivedDecoderManifest->getComplexSignalDecoderFormat( 123 ).mInterfaceId.empty() ); diff --git a/test/unit/TraceModuleTest.cpp b/test/unit/TraceModuleTest.cpp index 49b3f9e8..af789f3e 100644 --- a/test/unit/TraceModuleTest.cpp +++ b/test/unit/TraceModuleTest.cpp @@ -18,6 +18,8 @@ TEST( TraceModuleTest, TraceModulePrint ) TraceModule::get().setVariable( TraceVariable::READ_SOCKET_FRAMES_0, 20 ); TraceModule::get().setVariable( TraceVariable::READ_SOCKET_FRAMES_0, 15 ); + TraceModule::get().setVariable( TraceVariable::DATA_FORWARD_BYTES, 5 ); + TraceModule::get().sectionBegin( TraceSection::BUILD_MQTT ); std::this_thread::sleep_for( std::chrono::milliseconds( 4 ) ); TraceModule::get().sectionEnd( TraceSection::BUILD_MQTT ); diff --git a/test/unit/support/CollectionSchemeManagerMock.h b/test/unit/support/CollectionSchemeManagerMock.h index a40e0dfa..9d2bea7b 100644 --- a/test/unit/support/CollectionSchemeManagerMock.h +++ b/test/unit/support/CollectionSchemeManagerMock.h @@ -7,6 +7,7 @@ #include "CollectionSchemeIngestion.h" #include "CollectionSchemeIngestionList.h" #include "CollectionSchemeManager.h" +#include "CollectionSchemeManagerTest.h" #include "DecoderManifestIngestion.h" #include "Listener.h" #include @@ -30,26 +31,79 @@ using uint8Ptr = std::uint8_t *; using vectorUint8 = std::vector; using vectorICollectionSchemePtr = std::vector; -class mockCollectionSchemeManagerTest : public CollectionSchemeManager +class CollectionSchemeManagerWrapper : public CollectionSchemeManager { public: - mockCollectionSchemeManagerTest( std::string dm_id ) - : CollectionSchemeManager( dm_id ) + CollectionSchemeManagerWrapper( std::shared_ptr schemaPersistencyPtr, + CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr checkinSender, + std::shared_ptr rawDataBufferManager = nullptr ) + + : CollectionSchemeManager( schemaPersistencyPtr, canIDTranslator, checkinSender, rawDataBufferManager ) { } - mockCollectionSchemeManagerTest( std::string dm_id, - std::map &mapEnabled, - std::map &mapIdle ) - : CollectionSchemeManager( dm_id, mapEnabled, mapIdle ) + + CollectionSchemeManagerWrapper( std::shared_ptr schemaPersistencyPtr, + CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr checkinSender, + SyncID decoderManifestID, + std::shared_ptr rawDataBufferManager = nullptr ) + : CollectionSchemeManager( schemaPersistencyPtr, canIDTranslator, checkinSender, rawDataBufferManager ) { + mCurrentDecoderManifestID = decoderManifestID; + } + CollectionSchemeManagerWrapper( std::shared_ptr schemaPersistencyPtr, + CANInterfaceIDTranslator &canIDTranslator, + std::shared_ptr checkinSender, + SyncID decoderManifestID, + std::map &mapEnabled, + std::map &mapIdle, + std::shared_ptr rawDataBufferManager = nullptr ) + : CollectionSchemeManager( schemaPersistencyPtr, canIDTranslator, checkinSender, rawDataBufferManager ) + { + mCurrentDecoderManifestID = decoderManifestID; + mEnabledCollectionSchemeMap = mapEnabled; + mIdleCollectionSchemeMap = mapIdle; } - // void inspectionMatrixExtractor( const std::shared_ptr &inspectionMatrix ) override; - using sharedPtrInspectionMatrix = std::shared_ptr; - MOCK_METHOD( void, inspectionMatrixExtractor, (const sharedPtrInspectionMatrix &)); - // void inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) override; - using sharedPtrConstInspectionMatrix = std::shared_ptr; - MOCK_METHOD( void, inspectionMatrixUpdater, (const sharedPtrConstInspectionMatrix &)); + void + myInvokeCollectionScheme() + { + this->onCollectionSchemeUpdate( mPlTest ); + } + void + myInvokeDecoderManifest() + { + this->onDecoderManifestUpdate( mDmTest ); + } + void + updateAvailable() + { + CollectionSchemeManager::updateAvailable(); + } + + void + matrixExtractor( const std::shared_ptr &inspectionMatrix ) + { + CollectionSchemeManager::matrixExtractor( inspectionMatrix ); + } + + void + inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) + { + CollectionSchemeManager::inspectionMatrixUpdater( inspectionMatrix ); + } + +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + void + updateRawDataBufferConfigComplexSignals( + std::shared_ptr complexDataDecoderDictionary, + std::unordered_map &updatedSignals ) + { + CollectionSchemeManager::updateRawDataBufferConfigComplexSignals( complexDataDecoderDictionary, + updatedSignals ); + } +#endif void setCollectionSchemePersistency( const std::shared_ptr &collectionSchemePersistency ) @@ -80,16 +134,28 @@ class mockCollectionSchemeManagerTest : public CollectionSchemeManager return mTimeLine; } - bool - getmProcessCollectionScheme() + void + decoderDictionaryExtractor( + std::map> &decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + std::shared_ptr inspectionMatrix = nullptr +#endif + ) { - return mProcessCollectionScheme; + return CollectionSchemeManager::decoderDictionaryExtractor( decoderDictionaryMap +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + , + inspectionMatrix +#endif + ); } - bool - getmProcessDecoderManifest() + void + decoderDictionaryUpdater( + std::map> &decoderDictionaryMap ) { - return mProcessDecoderManifest; + return CollectionSchemeManager::decoderDictionaryUpdater( decoderDictionaryMap ); } bool @@ -110,10 +176,10 @@ class mockCollectionSchemeManagerTest : public CollectionSchemeManager return CollectionSchemeManager::checkTimeLine( currTime ); } - bool - sendCheckin() + void + updateCheckinDocuments() { - return CollectionSchemeManager::sendCheckin(); + CollectionSchemeManager::updateCheckinDocuments(); } void @@ -127,64 +193,123 @@ class mockCollectionSchemeManagerTest : public CollectionSchemeManager { return CollectionSchemeManager::retrieve( retrieveType ); } + + void + setmCollectionSchemeAvailable( bool val ) + { + mCollectionSchemeAvailable = val; + } + bool + getmCollectionSchemeAvailable() + { + return mCollectionSchemeAvailable; + } + + void + setmDecoderManifestAvailable( bool val ) + { + mDecoderManifestAvailable = val; + } + + bool + getmDecoderManifestAvailable() + { + return mDecoderManifestAvailable; + } + + void + setmProcessCollectionScheme( bool val ) + { + mProcessCollectionScheme = val; + } + + bool + getmProcessCollectionScheme() + { + return mProcessCollectionScheme; + } + + void + setmProcessDecoderManifest( bool val ) + { + mProcessDecoderManifest = val; + } + bool + getmProcessDecoderManifest() + { + return mProcessDecoderManifest; + } + +public: + IDecoderManifestPtr mDmTest; + std::shared_ptr mPlTest; }; class mockCollectionScheme : public CollectionSchemeIngestion { public: + mockCollectionScheme() + : CollectionSchemeIngestion( +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + ) + { + } + // bool build() override; - MOCK_METHOD( bool, build, () ); - // const std::string &getCollectionSchemeID() - MOCK_METHOD( const std::string &, getCollectionSchemeID, (), ( const ) ); + MOCK_METHOD( bool, build, (), ( override ) ); + // const SyncID &getCollectionSchemeID() + MOCK_METHOD( const SyncID &, getCollectionSchemeID, (), ( const, override ) ); - // virtual const std::string &getDecoderManifestID() const = 0; - MOCK_METHOD( const std::string &, getDecoderManifestID, (), ( const ) ); + // virtual const SyncID &getDecoderManifestID() const = 0; + MOCK_METHOD( const SyncID &, getDecoderManifestID, (), ( const, override ) ); // virtual uint64_t getStartTime() const = 0; - MOCK_METHOD( uint64_t, getStartTime, (), ( const ) ); + MOCK_METHOD( uint64_t, getStartTime, (), ( const, override ) ); // virtual uint64_t getExpiryTime() const = 0; - MOCK_METHOD( uint64_t, getExpiryTime, (), ( const ) ); + MOCK_METHOD( uint64_t, getExpiryTime, (), ( const, override ) ); }; class mockDecoderManifest : public DecoderManifestIngestion { public: - // virtual std::string getID() const = 0; - MOCK_METHOD( std::string, getID, (), ( const ) ); + // virtual SyncID getID() const = 0; + MOCK_METHOD( SyncID, getID, (), ( const, override ) ); // bool build() override; - MOCK_METHOD( bool, build, () ); + MOCK_METHOD( bool, build, (), ( override ) ); // bool copyData( const std::uint8_t *inputBuffer, const size_t size ) override; - MOCK_METHOD( bool, copyData, ( const std::uint8_t *, const size_t ) ); + MOCK_METHOD( bool, copyData, ( const std::uint8_t *, const size_t ), ( override ) ); // inline const std::vectorgetData() const override - MOCK_METHOD( const std::vector &, getData, (), ( const ) ); + MOCK_METHOD( const std::vector &, getData, (), ( const, override ) ); }; class mockCollectionSchemeList : public CollectionSchemeIngestionList { public: // bool build() override; - MOCK_METHOD( bool, build, () ); + MOCK_METHOD( bool, build, (), ( override ) ); // const std::vector &getCollectionSchemes() const override; - MOCK_METHOD( const std::vector &, getCollectionSchemes, (), ( const ) ); + MOCK_METHOD( const std::vector &, getCollectionSchemes, (), ( const, override ) ); - MOCK_METHOD( bool, copyData, ( const std::uint8_t *, const size_t ) ); - MOCK_METHOD( const std::vector &, getData, (), ( const ) ); + MOCK_METHOD( bool, copyData, ( const std::uint8_t *, const size_t ), ( override ) ); + MOCK_METHOD( const std::vector &, getData, (), ( const, override ) ); }; class mockCacheAndPersist : public CacheAndPersist { public: // ErrorCode write( const uint8_t *bufPtr, size_t size, DataType dataType, const std::string &filename ); - MOCK_METHOD( ErrorCode, write, (const uint8_t *, size_t, DataType, const std::string &)); + MOCK_METHOD( ErrorCode, write, (const uint8_t *, size_t, DataType, const std::string &), ( override ) ); // size_t getSize( DataType dataType, const std::string &filename ); - MOCK_METHOD( size_t, getSize, (DataType, const std::string &)); + MOCK_METHOD( size_t, getSize, (DataType, const std::string &), ( override ) ); // ErrorCode read( uint8_t *const readBufPtr, size_t size, DataType dataType, const std::string &filename ); - MOCK_METHOD( ErrorCode, read, (uint8_t *const, size_t, DataType, const std::string &)); + MOCK_METHOD( ErrorCode, read, (uint8_t *const, size_t, DataType, const std::string &), ( override ) ); }; } // namespace IoTFleetWise diff --git a/test/unit/support/CollectionSchemeManagerTest.h b/test/unit/support/CollectionSchemeManagerTest.h index ab05f753..9c342ec2 100644 --- a/test/unit/support/CollectionSchemeManagerTest.h +++ b/test/unit/support/CollectionSchemeManagerTest.h @@ -2,12 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once -#include "CacheAndPersist.h" #include "CollectionInspectionAPITypes.h" +#include "CollectionInspectionEngine.h" #include "CollectionInspectionWorkerThread.h" #include "CollectionSchemeIngestion.h" #include "CollectionSchemeIngestionList.h" -#include "CollectionSchemeManager.h" #include "DecoderManifestIngestion.h" #include "ICollectionScheme.h" #include "ICollectionSchemeList.h" @@ -16,14 +15,10 @@ #include "MessageTypes.h" #include "OBDOverCANModule.h" #include "SignalTypes.h" -#include "TimeTypes.h" #include "VehicleDataSourceTypes.h" #include // IWYU pragma: keep #include -#include -#include #include -#include #include // IWYU pragma: keep #include #include @@ -49,12 +44,12 @@ namespace IoTFleetWise class IDecoderManifestTest : public DecoderManifestIngestion { public: - IDecoderManifestTest( std::string id ) + IDecoderManifestTest( SyncID id ) : ID( id ) { } IDecoderManifestTest( - std::string id, + SyncID id, std::unordered_map> formatMap, std::unordered_map> signalToFrameAndNodeID, std::unordered_map signalIDToPIDDecoderFormat @@ -78,7 +73,7 @@ class IDecoderManifestTest : public DecoderManifestIngestion { } - std::string + SyncID getID() const { return ID; @@ -102,7 +97,11 @@ class IDecoderManifestTest : public DecoderManifestIngestion getNetworkProtocol( SignalID signalID ) const override { // a simple logic to assign network protocol type to signalID for testing purpose. - if ( signalID < 0x1000 ) + if ( signalID == INVALID_SIGNAL_ID ) + { + return VehicleDataSourceProtocol::INVALID_PROTOCOL; + } + else if ( signalID < 0x1000 ) { return VehicleDataSourceProtocol::RAW_SOCKET; } @@ -131,6 +130,20 @@ class IDecoderManifestTest : public DecoderManifestIngestion return NOT_FOUND_PID_DECODER_FORMAT; } + SignalType + getSignalType( SignalID signalID ) const override + { +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + if ( getNetworkProtocol( signalID ) == VehicleDataSourceProtocol::COMPLEX_DATA ) + { + return SignalType::UNKNOWN; + } +#else + static_cast( signalID ); +#endif + return SignalType::DOUBLE; + } + #ifdef FWE_FEATURE_VISION_SYSTEM_DATA ComplexSignalDecoderFormat getComplexSignalDecoderFormat( SignalID signalID ) const override @@ -156,7 +169,7 @@ class IDecoderManifestTest : public DecoderManifestIngestion #endif private: - std::string ID; + SyncID ID; std::unordered_map> mFormatMap; std::unordered_map> mSignalToFrameAndNodeID; std::unordered_map mSignalIDToPIDDecoderFormat; @@ -168,18 +181,23 @@ class IDecoderManifestTest : public DecoderManifestIngestion class ICollectionSchemeTest : public CollectionSchemeIngestion { public: - ICollectionSchemeTest( std::string collectionSchemeID, - std::string DMID, + ICollectionSchemeTest( SyncID collectionSchemeID, + SyncID DMID, uint64_t start, uint64_t stop, - Signals_t signalsIn, - RawCanFrames_t rawCanFrmsIn + const Signals_t &signalsIn, + const RawCanFrames_t &rawCanFrmsIn #ifdef FWE_FEATURE_VISION_SYSTEM_DATA , - PartialSignalIDLookup partialSignalLookup = PartialSignalIDLookup() + const PartialSignalIDLookup &partialSignalLookup = PartialSignalIDLookup() #endif ) - : collectionSchemeID( collectionSchemeID ) + : CollectionSchemeIngestion( +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + ) + , collectionSchemeID( collectionSchemeID ) , decoderManifestID( DMID ) , startTime( start ) , expiryTime( stop ) @@ -191,8 +209,8 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion , root( nullptr ) { } - ICollectionSchemeTest( std::string collectionSchemeID, - std::string DMID, + ICollectionSchemeTest( SyncID collectionSchemeID, + SyncID DMID, uint64_t start, uint64_t stop #ifdef FWE_FEATURE_VISION_SYSTEM_DATA @@ -200,7 +218,12 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion S3UploadMetadata s3UploadMetadata = S3UploadMetadata() #endif ) - : collectionSchemeID( collectionSchemeID ) + : CollectionSchemeIngestion( +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + ) + , collectionSchemeID( collectionSchemeID ) , decoderManifestID( DMID ) , startTime( start ) , expiryTime( stop ) @@ -210,21 +233,25 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion , root( nullptr ) { } - ICollectionSchemeTest( - std::string collectionSchemeID, std::string DMID, uint64_t start, uint64_t stop, ExpressionNode *root ) - : collectionSchemeID( collectionSchemeID ) + ICollectionSchemeTest( SyncID collectionSchemeID, SyncID DMID, uint64_t start, uint64_t stop, ExpressionNode *root ) + : CollectionSchemeIngestion( +#ifdef FWE_FEATURE_VISION_SYSTEM_DATA + std::make_shared() +#endif + ) + , collectionSchemeID( collectionSchemeID ) , decoderManifestID( DMID ) , startTime( start ) , expiryTime( stop ) , root( root ) { } - const std::string & + const SyncID & getCollectionSchemeID() const { return collectionSchemeID; } - const std::string & + const SyncID & getDecoderManifestID() const { return decoderManifestID; @@ -260,6 +287,12 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion return true; } + bool + isReady() const override + { + return true; + } + #ifdef FWE_FEATURE_VISION_SYSTEM_DATA const PartialSignalIDLookup & getPartialSignalIdToSignalPathLookupTable() const override @@ -274,8 +307,8 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion #endif private: - std::string collectionSchemeID; - std::string decoderManifestID; + SyncID collectionSchemeID; + SyncID decoderManifestID; uint64_t startTime; uint64_t expiryTime; Signals_t signals; @@ -290,7 +323,7 @@ class ICollectionSchemeTest : public CollectionSchemeIngestion class ICollectionSchemeListTest : public CollectionSchemeIngestionList { public: - ICollectionSchemeListTest( std::vector &list ) + ICollectionSchemeListTest( const std::vector &list ) : mCollectionSchemeTest( list ) { } @@ -305,6 +338,12 @@ class ICollectionSchemeListTest : public CollectionSchemeIngestionList return true; } + bool + isReady() const override + { + return true; + } + public: std::vector mCollectionSchemeTest; }; @@ -347,7 +386,8 @@ class CollectionInspectionWorkerThreadMock : public CollectionInspectionWorkerTh { public: CollectionInspectionWorkerThreadMock() - : mUpdateFlag( false ) + : CollectionInspectionWorkerThread( mEngine ) + , mInspectionMatrixUpdateFlag( false ) { } ~CollectionInspectionWorkerThreadMock() @@ -357,180 +397,23 @@ class CollectionInspectionWorkerThreadMock : public CollectionInspectionWorkerTh onChangeInspectionMatrix( const std::shared_ptr &inspectionMatrix ) { CollectionInspectionWorkerThread::onChangeInspectionMatrix( inspectionMatrix ); - mUpdateFlag = true; + mInspectionMatrixUpdateFlag = true; } void - setUpdateFlag( bool flag ) + setInspectionMatrixUpdateFlag( bool flag ) { - mUpdateFlag = flag; + mInspectionMatrixUpdateFlag = flag; } bool - getUpdateFlag() + getInspectionMatrixUpdateFlag() { - return mUpdateFlag; + return mInspectionMatrixUpdateFlag; } private: - // This flag is used for testing whether the listener received the update - bool mUpdateFlag; -}; - -class CollectionSchemeManagerTest : public CollectionSchemeManager -{ -public: - CollectionSchemeManagerTest() - { - } - CollectionSchemeManagerTest( std::string dm_id ) - : CollectionSchemeManager( dm_id ) - { - } - - void - myInvokeCollectionScheme() - { - this->onCollectionSchemeUpdate( mPlTest ); - } - void - myInvokeDecoderManifest() - { - this->onDecoderManifestUpdate( mDmTest ); - } - void - updateAvailable() - { - CollectionSchemeManager::updateAvailable(); - } - bool - rebuildMapsandTimeLine( const TimePoint &currTime ) - { - return ( CollectionSchemeManager::rebuildMapsandTimeLine( currTime ) ); - } - bool - updateMapsandTimeLine( const TimePoint &currTime ) - { - return CollectionSchemeManager::updateMapsandTimeLine( currTime ); - } - bool - checkTimeLine( const TimePoint &currTime ) - { - return ( CollectionSchemeManager::checkTimeLine( currTime ) ); - } - void - decoderDictionaryExtractor( - std::map> &decoderDictionaryMap ) - { - return CollectionSchemeManager::decoderDictionaryExtractor( decoderDictionaryMap ); - } - void - decoderDictionaryUpdater( - std::map> &decoderDictionaryMap ) - { - return CollectionSchemeManager::decoderDictionaryUpdater( decoderDictionaryMap ); - } - const std::priority_queue, std::greater> & - getTimeLine() - { - return CollectionSchemeManager::mTimeLine; - } - void - setDecoderManifest( IDecoderManifestPtr dm ) - { - CollectionSchemeManager::mDecoderManifest = dm; - } - void - setCollectionSchemeList( ICollectionSchemeListPtr pl ) - { - CollectionSchemeManager::mCollectionSchemeList = pl; - } - void - setCollectionSchemePersistency( std::shared_ptr pp ) - { - CollectionSchemeManager::mSchemaPersistency = pp; - } - void - inspectionMatrixExtractor( const std::shared_ptr &inspectionMatrix ) - { - CollectionSchemeManager::inspectionMatrixExtractor( inspectionMatrix ); - } - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - void - updateRawDataBufferConfig( - std::shared_ptr complexDataDecoderDictionary ) - { - CollectionSchemeManager::updateRawDataBufferConfig( complexDataDecoderDictionary ); - } -#endif - - void - inspectionMatrixUpdater( const std::shared_ptr &inspectionMatrix ) - { - CollectionSchemeManager::inspectionMatrixUpdater( inspectionMatrix ); - } - bool - retrieve( DataType retrieveType ) - { - return CollectionSchemeManager::retrieve( retrieveType ); - } - - void - store( DataType storeType ) - { - CollectionSchemeManager::store( storeType ); - } - - void - setmCollectionSchemeAvailable( bool val ) - { - mCollectionSchemeAvailable = val; - } - bool - getmCollectionSchemeAvailable() - { - return mCollectionSchemeAvailable; - } - - void - setmDecoderManifestAvailable( bool val ) - { - mDecoderManifestAvailable = val; - } - - bool - getmDecoderManifestAvailable() - { - return mDecoderManifestAvailable; - } - - void - setmProcessCollectionScheme( bool val ) - { - mProcessCollectionScheme = val; - } - - bool - getmProcessCollectionScheme() - { - return mProcessCollectionScheme; - } - - void - setmProcessDecoderManifest( bool val ) - { - mProcessDecoderManifest = val; - } - bool - getmProcessDecoderManifest() - { - return mProcessDecoderManifest; - } - -public: - IDecoderManifestPtr mDmTest; - std::shared_ptr mPlTest; - -protected: + CollectionInspectionEngine mEngine; + // This flag is used for testing whether the listener received the Inspection Matrix update + bool mInspectionMatrixUpdateFlag; }; } // namespace IoTFleetWise diff --git a/test/unit/support/ConnectivityModuleMock.h b/test/unit/support/ConnectivityModuleMock.h index eef258a1..405c8f79 100644 --- a/test/unit/support/ConnectivityModuleMock.h +++ b/test/unit/support/ConnectivityModuleMock.h @@ -17,25 +17,29 @@ namespace Testing class ConnectivityModuleMock : public IConnectivityModule { public: - MOCK_METHOD( bool, isAlive, (), ( const override ) ); + MOCK_METHOD( bool, isAlive, (), ( const, override ) ); - std::shared_ptr - createNewChannel( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription = false ) override + std::shared_ptr + createSender( const std::string &topicName, QoS publishQoS = QoS::AT_MOST_ONCE ) override { - return mockedCreateNewChannel( payloadManager, topicName, subscription ); + return mockedCreateSender( topicName, publishQoS ); }; - MOCK_METHOD( std::shared_ptr, - mockedCreateNewChannel, - ( const std::shared_ptr &payloadManager, - const std::string &topicName, - bool subscription ) ); + MOCK_METHOD( std::shared_ptr, mockedCreateSender, ( const std::string &topicName, QoS publishQoS ) ); + + std::shared_ptr + createReceiver( const std::string &topicName ) override + { + return mockedCreateReceiver( topicName ); + }; + + MOCK_METHOD( std::shared_ptr, mockedCreateReceiver, ( const std::string &topicName ) ); MOCK_METHOD( bool, disconnect, (), ( override ) ); MOCK_METHOD( bool, connect, (), ( override ) ); + + MOCK_METHOD( void, subscribeToConnectionEstablished, ( OnConnectionEstablishedCallback callback ), ( override ) ); }; } // namespace Testing diff --git a/test/unit/support/DataSenderIonWriterMock.h b/test/unit/support/DataSenderIonWriterMock.h index 7f2196bc..793c2cfd 100644 --- a/test/unit/support/DataSenderIonWriterMock.h +++ b/test/unit/support/DataSenderIonWriterMock.h @@ -27,7 +27,7 @@ class DataSenderIonWriterMock : public DataSenderIonWriter MOCK_METHOD( void, setupVehicleData, - ( const TriggeredCollectionSchemeDataPtr &mTriggeredCollectionSchemeData ), + ( std::shared_ptr mTriggeredVisionSystemData ), ( override ) ); MOCK_METHOD( std::unique_ptr, getStreambufBuilder, (), ( override ) ); diff --git a/test/unit/support/DataSenderManagerMock.h b/test/unit/support/DataSenderManagerMock.h index 61d3ee2d..5996f323 100644 --- a/test/unit/support/DataSenderManagerMock.h +++ b/test/unit/support/DataSenderManagerMock.h @@ -21,76 +21,50 @@ class DataSenderManagerMock : public DataSenderManager public: unsigned mCheckAndSendRetrievedDataCalls{ 0 }; - DataSenderManagerMock( CANInterfaceIDTranslator &canIDTranslator ) - : DataSenderManager( nullptr, - nullptr, - canIDTranslator, - 0 -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - nullptr, - nullptr, - "" -#endif - ) + DataSenderManagerMock() + : DataSenderManager( {}, nullptr, nullptr ) { } void - processCollectedData( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - std::function reportUploadCallback -#endif - ) override + processData( std::shared_ptr data ) override { std::lock_guard lock( mProcessedDataMutex ); - mProcessedData.push_back( triggeredCollectionSchemeDataPtr ); - mockedProcessCollectedData( triggeredCollectionSchemeDataPtr -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - , - reportUploadCallback -#endif - ); + mProcessedData.push_back( data ); + mockedProcessData( data ); } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - MOCK_METHOD( void, - mockedProcessCollectedData, - ( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr, - std::function reportUploadCallback ) ); -#else - MOCK_METHOD( void, - mockedProcessCollectedData, - ( const TriggeredCollectionSchemeDataPtr triggeredCollectionSchemeDataPtr ) ); -#endif + MOCK_METHOD( void, mockedProcessData, (std::shared_ptr)); - std::vector + std::vector> getProcessedData() { std::lock_guard lock( mProcessedDataMutex ); return mProcessedData; } - void - checkAndSendRetrievedData() override + template + std::vector> + getProcessedData() { - mCheckAndSendRetrievedDataCalls++; + std::lock_guard lock( mProcessedDataMutex ); + std::vector> castedProcessedData; + for ( auto data : mProcessedData ) + { + castedProcessedData.push_back( std::dynamic_pointer_cast( data ) ); + } + return castedProcessedData; } -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - std::shared_ptr mActiveCollectionSchemes; void - onChangeCollectionSchemeList( - const std::shared_ptr &activeCollectionSchemes ) override + checkAndSendRetrievedData() override { - mActiveCollectionSchemes = activeCollectionSchemes; + mCheckAndSendRetrievedDataCalls++; } -#endif private: // Record the calls so that we can wait for asynchronous calls to happen. - std::vector mProcessedData; + std::vector> mProcessedData; std::mutex mProcessedDataMutex; }; diff --git a/test/unit/support/PayloadManagerMock.h b/test/unit/support/PayloadManagerMock.h index fb6053a2..28e40742 100644 --- a/test/unit/support/PayloadManagerMock.h +++ b/test/unit/support/PayloadManagerMock.h @@ -22,36 +22,7 @@ class PayloadManagerMock : public PayloadManager PayloadManagerMock() : PayloadManager( nullptr ){}; -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - MOCK_METHOD( bool, - storeData, - ( const std::uint8_t *buf, - size_t size, - const CollectionSchemeParams &collectionSchemeParams, - const struct S3UploadParams &s3UploadParams ), - ( override ) ); -#else - MOCK_METHOD( bool, - storeData, - ( const std::uint8_t *buf, size_t size, const CollectionSchemeParams &collectionSchemeParams ), - ( override ) ); - -#endif - -#ifdef FWE_FEATURE_VISION_SYSTEM_DATA - MOCK_METHOD( void, - storeMetadata, - ( const std::string filename, - size_t size, - const CollectionSchemeParams &collectionSchemeParams, - const struct S3UploadParams &s3UploadParams ), - ( override ) ); -#else - MOCK_METHOD( void, - storeMetadata, - ( const std::string filename, size_t size, const CollectionSchemeParams &collectionSchemeParams ), - ( override ) ); -#endif + MOCK_METHOD( bool, storeData, ( const std::uint8_t *buf, size_t size, const Json::Value &metadata ), ( override ) ); MOCK_METHOD( ErrorCode, retrievePayloadMetadata, ( Json::Value & files ), ( override ) ); diff --git a/test/unit/support/RawDataBufferManagerSpy.h b/test/unit/support/RawDataBufferManagerSpy.h index c76eb53d..cb4b2be0 100644 --- a/test/unit/support/RawDataBufferManagerSpy.h +++ b/test/unit/support/RawDataBufferManagerSpy.h @@ -52,7 +52,8 @@ class RawDataBufferManagerSpy : public RawData::BufferManager RawData::BufferHandleUsageStage handleUsageStage ) ); RawData::BufferErrorCode - updateConfig( const std::unordered_map &updatedSignals ) + updateConfig( + const std::unordered_map &updatedSignals ) override { mockedUpdateConfig( updatedSignals ); return RawData::BufferManager::updateConfig( updatedSignals ); @@ -61,6 +62,15 @@ class RawDataBufferManagerSpy : public RawData::BufferManager MOCK_METHOD( RawData::BufferErrorCode, mockedUpdateConfig, ( ( const std::unordered_map &updatedSignals ) ) ); + + RawData::LoanedFrame + borrowFrame( RawData::BufferTypeId typeId, RawData::BufferHandle handle ) override + { + mockedBorrowFrame( typeId, handle ); + return RawData::BufferManager::borrowFrame( typeId, handle ); + } + + MOCK_METHOD( void, mockedBorrowFrame, ( RawData::BufferTypeId typeId, RawData::BufferHandle handle ) ); }; } // namespace Testing diff --git a/test/unit/support/S3SenderMock.h b/test/unit/support/S3SenderMock.h index 74cf1760..ecbc37c4 100644 --- a/test/unit/support/S3SenderMock.h +++ b/test/unit/support/S3SenderMock.h @@ -21,16 +21,16 @@ class S3SenderMock : public S3Sender { public: S3SenderMock() - : S3Sender( nullptr, nullptr, 0 ) + : S3Sender( nullptr, 0 ) { } - MOCK_METHOD( ConnectivityError, + MOCK_METHOD( void, sendStream, ( std::unique_ptr streambufBuilder, const S3UploadMetadata &uploadMetadata, const std::string &objectKey, - std::function resultCallback ), + ResultCallback resultCallback ), ( override ) ); }; diff --git a/test/unit/support/SenderMock.h b/test/unit/support/SenderMock.h index 93c32b9b..7a394267 100644 --- a/test/unit/support/SenderMock.h +++ b/test/unit/support/SenderMock.h @@ -24,19 +24,31 @@ class SenderMock : public ISender struct SentBufferData { std::string data; - CollectionSchemeParams collectionSchemeParams; + OnDataSentCallback callback; }; MOCK_METHOD( bool, isAlive, (), ( override ) ); MOCK_METHOD( size_t, getMaxSendSize, (), ( const, override ) ); - ConnectivityError - sendBuffer( const std::uint8_t *buf, size_t size, CollectionSchemeParams collectionSchemeParams ) override + void + sendBuffer( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) override { std::lock_guard lock( mSentBufferDataMutex ); - mSentBufferData.push_back( SentBufferData{ std::string( buf, buf + size ), collectionSchemeParams } ); - return mockedSendBuffer( buf, size, collectionSchemeParams ); + mSentBufferData.push_back( SentBufferData{ std::string( buf, buf + size ), callback } ); + mockedSendBuffer( buf, size, callback ); + } + + void + sendBufferToTopic( __attribute__( ( unused ) ) const std::string &topic, + const std::uint8_t *buf, + size_t size, + OnDataSentCallback callback ) override + { + // TODO: Make a map of topic to SentBufferData. For now, mark topic parameter as unused + std::lock_guard lock( mSentBufferDataMutex ); + mSentBufferData.push_back( SentBufferData{ std::string( buf, buf + size ), callback } ); + mockedSendBuffer( buf, size, callback ); } std::vector @@ -46,14 +58,16 @@ class SenderMock : public ISender return mSentBufferData; } - MOCK_METHOD( ConnectivityError, - mockedSendBuffer, - ( const std::uint8_t *buf, size_t size, CollectionSchemeParams collectionSchemeParams ) ); + void + clearSentBufferData() + { + std::lock_guard lock( mSentBufferDataMutex ); + mSentBufferData.clear(); + } + + MOCK_METHOD( void, mockedSendBuffer, ( const std::uint8_t *buf, size_t size, OnDataSentCallback callback ) ); - MOCK_METHOD( ConnectivityError, - sendFile, - ( const std::string &filePath, size_t size, CollectionSchemeParams collectionSchemeParams ), - ( override ) ); + MOCK_METHOD( unsigned, getPayloadCountSent, (), ( const, override ) ); private: // Record the calls so that we can wait for asynchronous calls to happen. diff --git a/test/unit/support/Testing.h b/test/unit/support/Testing.h index 870a4015..aa6649a8 100644 --- a/test/unit/support/Testing.h +++ b/test/unit/support/Testing.h @@ -3,9 +3,11 @@ #pragma once +#include "CacheAndPersist.h" #include "CollectionInspectionAPITypes.h" #include "SignalTypes.h" #include "TimeTypes.h" +#include #include #include #include @@ -209,36 +211,10 @@ assertSignalValue( const SignalValueWrapper &signalValueWrapper, * @return A string that can be used as the parameter name */ inline std::string -signalTypeToString( const testing::TestParamInfo &info ) +signalTypeParamInfoToString( const testing::TestParamInfo &info ) { SignalType signalType = info.param; - switch ( signalType ) - { - case SignalType::UINT8: - return "UINT8"; - case SignalType::INT8: - return "INT8"; - case SignalType::UINT16: - return "UINT16"; - case SignalType::INT16: - return "INT16"; - case SignalType::UINT32: - return "UINT32"; - case SignalType::INT32: - return "INT32"; - case SignalType::UINT64: - return "UINT64"; - case SignalType::INT64: - return "INT64"; - case SignalType::FLOAT: - return "FLOAT"; - case SignalType::DOUBLE: - return "DOUBLE"; - case SignalType::BOOLEAN: - return "BOOLEAN"; - default: - throw std::invalid_argument( "Unsupported signal type" ); - } + return signalTypeToString( signalType ); } constexpr inline std::size_t operator""_KiB( unsigned long long sizeBytes ) @@ -256,5 +232,27 @@ constexpr inline std::size_t operator""_GiB( unsigned long long sizeBytes ) return static_cast( sizeBytes * 1024 * 1024 * 1024 ); } +inline boost::filesystem::path +getTempDir() +{ + auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + auto testDir = boost::filesystem::current_path() / "unit_tests_tmp" / testInfo->test_case_name() / testInfo->name(); + boost::filesystem::create_directories( testDir ); + return testDir; +} + +inline std::shared_ptr +createCacheAndPersist() +{ + auto cacheAndPersist = std::make_shared( getTempDir().string(), 131072 ); + if ( !cacheAndPersist->init() ) + { + // throw instead of using ASSERT_TRUE because the caller would have to use ASSERT_NO_FATAL_FAILURE + // in order for the assert to work. + throw std::runtime_error( "Failed to initialize persistency" ); + } + return cacheAndPersist; +} + } // namespace IoTFleetWise } // namespace Aws diff --git a/test/unit/support/TransferManagerWrapperMock.h b/test/unit/support/TransferManagerWrapperMock.h index 94e37348..ca6984af 100644 --- a/test/unit/support/TransferManagerWrapperMock.h +++ b/test/unit/support/TransferManagerWrapperMock.h @@ -69,12 +69,14 @@ class TransferManagerWrapperMock : public TransferManagerWrapper MOCK_METHOD( std::shared_ptr, RetryUpload, - (const Aws::String &, const std::shared_ptr &)); + (const Aws::String &, const std::shared_ptr &), + ( override ) ); MOCK_METHOD( std::shared_ptr, RetryUpload, ( const std::shared_ptr &stream, - const std::shared_ptr &retryHandle ) ); + const std::shared_ptr &retryHandle ), + ( override ) ); }; } // namespace IoTFleetWise diff --git a/test/unit/support/static-config-ok.json b/test/unit/support/static-config-ok.json index bfb44000..94bcfc20 100644 --- a/test/unit/support/static-config-ok.json +++ b/test/unit/support/static-config-ok.json @@ -5,7 +5,8 @@ "canInterface": { "interfaceName": "vcan0", "protocolName": "CAN", - "protocolVersion": "2.0A" + "protocolVersion": "2.0A", + "timestampType": "Software" }, "interfaceId": "1", "type": "canInterface" @@ -53,7 +54,8 @@ "internalParameters": { "readyToPublishDataBufferSize": 10000, "systemWideLogLevel": "Trace", - "persistencyUploadRetryIntervalMs": 5000 + "persistencyUploadRetryIntervalMs": 5000, + "maximumAwsSdkHeapMemoryBytes": 10000000 }, "publishToCloudParameters": { "maxPublishMessageCount": 1000, @@ -84,6 +86,24 @@ "metricsUploadIntervalMs": 60000, "loggingUploadMaxWaitBeforeUploadMs": 60000, "profilerPrefix": "TestVehicle1" + }, + "credentialsProvider": { + "endpointUrl": "my-endpoint.my-region.amazonaws.com", + "roleAlias": "my-role-alias" + }, + "s3Upload": { + "maxConnections": 1, + "multipartSize": 123 + }, + "visionSystemDataCollection": { + "rawDataBuffer": { + "overridesPerSignal": [ + { + "interfaceId": "my-vsd-interface", + "messageId": "my-message" + } + ] + } } } } diff --git a/test/unit/support/valgrind.supp b/test/unit/support/valgrind.supp index 01c7aa5e..e69de29b 100644 --- a/test/unit/support/valgrind.supp +++ b/test/unit/support/valgrind.supp @@ -1,55 +0,0 @@ -{ - - Memcheck:Param - write(buf) - ... - fun:sem_open - ... -} -{ - aws_io_library_init - Memcheck:Leak - match-leak-kinds: definite,reachable - ... - fun:aws_io_library_init - ... -} -{ - aws_mqtt_library_init - Memcheck:Leak - match-leak-kinds: definite,reachable - ... - fun:aws_mqtt_library_init - ... -} -{ - __libc_csu_init - Memcheck:Leak - match-leak-kinds: reachable - ... - fun:__libc_csu_init - ... -} -{ - boost::lockfree::do_push - Memcheck:Cond - ... - fun:_ZN5boost8lockfree5queue* - ... -} -{ - _dl_init - Memcheck:Leak - match-leak-kinds: reachable - ... - fun:_dl_init - ... -} -{ - _dl_open - Memcheck:Leak - match-leak-kinds: reachable - ... - fun:_dl_open - ... -} diff --git a/tools/android-app/README.md b/tools/android-app/README.md index 3aa334fe..b3ab78e6 100644 --- a/tools/android-app/README.md +++ b/tools/android-app/README.md @@ -57,15 +57,6 @@ available [ELM327 Bluetooth OBD adapter](https://www.amazon.com/s?k=elm327+bluet aws s3 rm s3:///fwdemo-android--creds.json ``` -1. In the cloud shell, run the following command to create an AWS IoT FleetWise campaign to collect - GPS and OBD PID data and send it to Amazon Timestream. After a short time you should see that the - status in the app is updated with: - `Campaign ARNs: arn:aws:iotfleetwise:us-east-1:XXXXXXXXXXXX:campaign/fwdemo-android-XXXXXXXXXX-campaign`. - - ```bash - ./setup-iotfleetwise.sh - ``` - 1. Plug in the ELM327 OBD Bluetooth adapter to your vehicle and switch on the ignition. 1. Go to the Bluetooth menu of your phone, then pair the ELM327 Bluetooth adapter. Typically the the @@ -76,20 +67,40 @@ available [ELM327 Bluetooth OBD adapter](https://www.amazon.com/s?k=elm327+bluet 1. After a short time you should see that the status is updated with: `Bluetooth: Connected to ELM327 vX.X` and `Supported OBD PIDs: XX XX XX`. +1. In the cloud shell, run the following command to create an AWS IoT FleetWise campaign to collect + GPS and OBD PID data and send it to Amazon Timestream. After a short time you should see that the + status in the app is updated with: + `Campaign ARNs: arn:aws:iotfleetwise:us-east-1:XXXXXXXXXXXX:campaign/fwdemo-android-XXXXXXXXXX-campaign`. + + ```bash + ../../cloud/demo.sh \ + --vehicle-name `cat config/vehicle-name.txt` \ + --node-file ../../cloud/obd-nodes.json \ + --node-file externalGpsNodes.json\ + --decoder-file ../../cloud/obd-decoders.json \ + --decoder-file externalGpsDecoders.json \ + --network-interface-file ../../cloud/network-interface-obd.json \ + --network-interface-file network-interface-can-external-gps.json \ + --campaign-file campaign-android-obd.json + ``` + 1. Go to the [Amazon Timestream Query editor](https://us-east-1.console.aws.amazon.com/timestream/home?region=us-east-1#query-editor:) - and run the query suggested by the `setup-iotfleetwise.sh` script, which will be of the form: + and run the query suggested by the demo script, which will be of the form: ``` - SELECT * FROM "IoTFleetWiseDB-"."VehicleDataTable" WHERE vehicleName='fwdemo-android-' ORDER BY time DESC LIMIT 1000 + SELECT * FROM "IoTFleetWiseDB-"."VehicleDataTable" WHERE vehicleName='fwdemo-android-' AND time between ago(1m) and now() ORDER BY time ASC ``` -1. **Optional:** If you want to clean up the resources created by the `provision.sh` and - `setup-iotfleetwise.sh` scripts, run the following command. **Note:** this will not delete the - Amazon Timestream database. +1. **Optional:** If you want to clean up the resources created by the `provision.sh` and `demo.sh` + scripts, run the following command. **Note:** this will not delete the Amazon Timestream + database. ```bash - ./clean-up.sh + ../../cloud/clean-up.sh \ + && ../../provision.sh \ + --vehicle-name `cat config/vehicle-name.txt` \ + --only-clean-up ``` ## Android Automotive User Guide @@ -157,23 +168,35 @@ privileged VHAL properties. To demonstrate the app accessing privileged VHAL pro `Campaign ARNs: arn:aws:iotfleetwise:us-east-1:XXXXXXXXXXXX:campaign/fwdemo-android-XXXXXXXXXX-campaign`. ```bash - ./setup-iotfleetwise.sh + sudo -H ../../cloud/install-deps.sh \ + && ../../cloud/demo.sh \ + --vehicle-name `cat config/vehicle-name.txt` \ + --node-file externalGpsNodes.json\ + --node-file aaosVhalNodes.json \ + --decoder-file externalGpsDecoders.json \ + --decoder-file aaosVhalDecoders.json \ + --network-interface-file network-interface-can-external-gps.json \ + --network-interface-file network-interface-can-aaos-vhal.json \ + --campaign-file campaign-android-aaos-vhal.json ``` 1. Go to the [Amazon Timestream Query editor](https://us-east-1.console.aws.amazon.com/timestream/home?region=us-east-1#query-editor:) - and run the query suggested by the `setup-iotfleetwise.sh` script, which will be of the form: + and run the query suggested by the script, which will be of the form: ``` - SELECT * FROM "IoTFleetWiseDB-"."VehicleDataTable" WHERE vehicleName='fwdemo-android-' ORDER BY time DESC LIMIT 1000 + SELECT * FROM "IoTFleetWiseDB-"."VehicleDataTable" WHERE vehicleName='fwdemo-android-' AND time between ago(1m) and now() ORDER BY time ASC ``` -1. **Optional:** If you want to clean up the resources created by the `provision.sh` and - `setup-iotfleetwise.sh` scripts, run the following command. **Note:** this will not delete the - Amazon Timestream database. +1. **Optional:** If you want to clean up the resources created by the `provision.sh` and `demo.sh` + scripts, run the following command. **Note:** this will not delete the Amazon Timestream + database. ```bash - ./clean-up.sh + ../../cloud/clean-up.sh \ + && ../../provision.sh \ + --vehicle-name `cat config/vehicle-name.txt` \ + --only-clean-up ``` ## Android Developer Guide diff --git a/tools/android-app/app/src/main/java/com/aws/iotfleetwise/Fwe.java b/tools/android-app/app/src/main/java/com/aws/iotfleetwise/Fwe.java index 7f499858..a2202fdd 100644 --- a/tools/android-app/app/src/main/java/com/aws/iotfleetwise/Fwe.java +++ b/tools/android-app/app/src/main/java/com/aws/iotfleetwise/Fwe.java @@ -73,7 +73,7 @@ public native static int run( * @param signalId Signal ID * @param value Vehicle property value */ - public native static void setVehicleProperty(int signalId, double value); + public native static void setVehicleProperty(int signalId, Object value); /** * Get a status summary diff --git a/tools/android-app/cloud/.gitignore b/tools/android-app/cloud/.gitignore index 07bffee7..cf8c338c 100644 --- a/tools/android-app/cloud/.gitignore +++ b/tools/android-app/cloud/.gitignore @@ -2,3 +2,4 @@ config/ aaos-vhal-types.h aaos-vhal-fqns.txt fwdemo-android-*-timestream-result.json +demo.env diff --git a/tools/android-app/cloud/campaign-android.json b/tools/android-app/cloud/campaign-android-aaos-vhal.json similarity index 74% rename from tools/android-app/cloud/campaign-android.json rename to tools/android-app/cloud/campaign-android-aaos-vhal.json index 88604f9a..f995fcd7 100644 --- a/tools/android-app/cloud/campaign-android.json +++ b/tools/android-app/cloud/campaign-android-aaos-vhal.json @@ -14,81 +14,6 @@ { "name": "Vehicle.CurrentLocation.Longitude" }, - { - "name": "Vehicle.OBD.EngineLoad" - }, - { - "name": "Vehicle.OBD.CoolantTemperature" - }, - { - "name": "Vehicle.OBD.FuelPressure" - }, - { - "name": "Vehicle.OBD.EngineSpeed" - }, - { - "name": "Vehicle.OBD.Speed" - }, - { - "name": "Vehicle.OBD.IntakeTemp" - }, - { - "name": "Vehicle.OBD.ThrottlePosition" - }, - { - "name": "Vehicle.OBD.RunTime" - }, - { - "name": "Vehicle.OBD.FuelLevel" - }, - { - "name": "Vehicle.OBD.BarometricPressure" - }, - { - "name": "Vehicle.OBD.ControlModuleVoltage" - }, - { - "name": "Vehicle.OBD.AbsoluteLoad" - }, - { - "name": "Vehicle.OBD.AmbientAirTemperature" - }, - { - "name": "Vehicle.OBD.FuelType" - }, - { - "name": "Vehicle.OBD.HybridBatteryRemaining" - }, - { - "name": "Vehicle.OBD.OilTemperature" - }, - { - "name": "Vehicle.OBD.FuelRate" - }, - { - "name": "Vehicle.OBD.DemandEngineTorque" - }, - { - "name": "Vehicle.OBD.ActualEngineTorque" - }, - { - "name": "Vehicle.OBD.MafA" - }, - { - "name": "Vehicle.OBD.MafB" - }, - { - "name": "Vehicle.OBD.MAF" - }, - { - "name": "Vehicle.OBD.TransmissionActualGearStatusSupport" - }, - { - "name": "Vehicle.OBD.TransmissionActualGearRatio" - }, - { - "name": "Vehicle.OBD.Odometer" - }, { "name": "Vehicle.VHAL.PERF_ODOMETER" }, diff --git a/tools/android-app/cloud/campaign-android-obd.json b/tools/android-app/cloud/campaign-android-obd.json new file mode 100644 index 00000000..ee9a7de2 --- /dev/null +++ b/tools/android-app/cloud/campaign-android-obd.json @@ -0,0 +1,93 @@ +{ + "compression": "SNAPPY", + "diagnosticsMode": "SEND_ACTIVE_DTCS", + "spoolingMode": "TO_DISK", + "collectionScheme": { + "timeBasedCollectionScheme": { + "periodMs": 10000 + } + }, + "signalsToCollect": [ + { + "name": "Vehicle.CurrentLocation.Latitude" + }, + { + "name": "Vehicle.CurrentLocation.Longitude" + }, + { + "name": "Vehicle.OBD.EngineLoad" + }, + { + "name": "Vehicle.OBD.CoolantTemperature" + }, + { + "name": "Vehicle.OBD.FuelPressure" + }, + { + "name": "Vehicle.OBD.EngineSpeed" + }, + { + "name": "Vehicle.OBD.Speed" + }, + { + "name": "Vehicle.OBD.IntakeTemp" + }, + { + "name": "Vehicle.OBD.ThrottlePosition" + }, + { + "name": "Vehicle.OBD.RunTime" + }, + { + "name": "Vehicle.OBD.FuelLevel" + }, + { + "name": "Vehicle.OBD.BarometricPressure" + }, + { + "name": "Vehicle.OBD.ControlModuleVoltage" + }, + { + "name": "Vehicle.OBD.AbsoluteLoad" + }, + { + "name": "Vehicle.OBD.AmbientAirTemperature" + }, + { + "name": "Vehicle.OBD.FuelType" + }, + { + "name": "Vehicle.OBD.HybridBatteryRemaining" + }, + { + "name": "Vehicle.OBD.OilTemperature" + }, + { + "name": "Vehicle.OBD.FuelRate" + }, + { + "name": "Vehicle.OBD.DemandEngineTorque" + }, + { + "name": "Vehicle.OBD.ActualEngineTorque" + }, + { + "name": "Vehicle.OBD.MafA" + }, + { + "name": "Vehicle.OBD.MafB" + }, + { + "name": "Vehicle.OBD.MAF" + }, + { + "name": "Vehicle.OBD.TransmissionActualGearStatusSupport" + }, + { + "name": "Vehicle.OBD.TransmissionActualGearRatio" + }, + { + "name": "Vehicle.OBD.Odometer" + } + ] +} diff --git a/tools/android-app/cloud/clean-up.sh b/tools/android-app/cloud/clean-up.sh deleted file mode 100755 index 5ce390b0..00000000 --- a/tools/android-app/cloud/clean-up.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -ENDPOINT_URL="" -ENDPOINT_URL_OPTION="" -REGION="us-east-1" -VEHICLE_NAME_FILENAME="config/vehicle-name.txt" -VEHICLE_NAME="" -THING_POLICY_FILENAME="config/thing-policy.txt" -THING_POLICY="" -SERVICE_ROLE_ARN_FILENAME="config/service-role-arn.txt" -SERVICE_ROLE_ARN="" -TIMESTAMP_FILENAME="config/timestamp.txt" -TIMESTAMP="" - -parse_args() { - while [ "$#" -gt 0 ]; do - case $1 in - --vehicle-name) - VEHICLE_NAME=$2 - shift - ;; - --timestamp) - TIMESTAMP=$2 - shift - ;; - --service-role-arn) - SERVICE_ROLE_ARN=$2 - shift - ;; - --thing-policy) - THING_POLICY=$2 - shift - ;; - --endpoint-url) - ENDPOINT_URL=$2 - shift - ;; - --region) - REGION=$2 - shift - ;; - --help) - echo "Usage: $0 [OPTION]" - echo " --vehicle-name Vehicle name, by default will be read from ${VEHICLE_NAME_FILENAME}" - echo " --timestamp Timestamp of setup-iotfleetwise.sh script, by default will be read from ${TIMESTAMP_FILENAME}" - echo " --service-role-arn Service role ARN for accessing Timestream, by default will be read from ${SERVICE_ROLE_ARN_FILENAME}" - echo " --thing-policy Name of IoT thing policy, by default will be read from ${THING_POLICY_FILENAME}" - echo " --endpoint-url The endpoint URL used for AWS CLI calls" - echo " --region The region used for AWS CLI calls, default: ${REGION}" - exit 0 - ;; - esac - shift - done - - if [ "${ENDPOINT_URL}" != "" ]; then - ENDPOINT_URL_OPTION="--endpoint-url ${ENDPOINT_URL}" - fi - - if [ "${VEHICLE_NAME}" == "" ]; then - if [ -f "${VEHICLE_NAME_FILENAME}" ]; then - VEHICLE_NAME=`cat ${VEHICLE_NAME_FILENAME}` - else - echo "Error: vehicle name not provided" - exit -1 - fi - fi - - if [ "${TIMESTAMP}" == "" ]; then - if [ -f "${TIMESTAMP_FILENAME}" ]; then - TIMESTAMP=`cat ${TIMESTAMP_FILENAME}` - fi - fi - - if [ "${SERVICE_ROLE_ARN}" == "" ]; then - if [ -f "${SERVICE_ROLE_ARN_FILENAME}" ]; then - SERVICE_ROLE_ARN=`cat ${SERVICE_ROLE_ARN_FILENAME}` - fi - fi - - if [ "${THING_POLICY}" == "" ]; then - if [ -f "${THING_POLICY_FILENAME}" ]; then - THING_POLICY=`cat ${THING_POLICY_FILENAME}` - fi - fi -} - -echo "=========================================" -echo "AWS IoT FleetWise Android Clean-up Script" -echo "=========================================" - -parse_args "$@" - -echo "Getting AWS account ID..." -ACCOUNT_ID=`aws sts get-caller-identity --query "Account" --output text` -echo ${ACCOUNT_ID} - -if [ "${TIMESTAMP}" != "" ]; then - NAME="${VEHICLE_NAME}-${TIMESTAMP}" - - echo "Suspending campaign..." - aws iotfleetwise update-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign \ - --action SUSPEND | jq -r .arn || true - - echo "Deleting campaign..." - aws iotfleetwise delete-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign | jq -r .arn || true - - echo "Disassociating vehicle ${VEHICLE_NAME}..." - aws iotfleetwise disassociate-vehicle-fleet \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --fleet-id ${NAME}-fleet \ - --vehicle-name "${VEHICLE_NAME}" || true - - echo "Deleting vehicle ${VEHICLE_NAME}..." - aws iotfleetwise delete-vehicle \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --vehicle-name "${VEHICLE_NAME}" | jq -r .arn || true - - echo "Deleting fleet..." - aws iotfleetwise delete-fleet \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --fleet-id ${NAME}-fleet | jq -r .arn || true - - echo "Deleting decoder manifest..." - aws iotfleetwise delete-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest | jq -r .arn || true - - echo "Deleting model manifest..." - aws iotfleetwise delete-model-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest | jq -r .arn || true - - rm -f ${TIMESTAMP_FILENAME} -fi - -if [ "${SERVICE_ROLE_ARN}" != "" ]; then - SERVICE_ROLE=`echo "${SERVICE_ROLE_ARN}" | cut -d "/" -f 2` - SERVICE_ROLE_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/${SERVICE_ROLE}-policy" - echo "Detatching policy..." - aws iam detach-role-policy --role-name ${SERVICE_ROLE} --policy-arn ${SERVICE_ROLE_POLICY_ARN} --region ${REGION} || true - echo "Deleting policy..." - aws iam delete-policy --policy-arn ${SERVICE_ROLE_POLICY_ARN} --region ${REGION} || true - echo "Deleting role..." - aws iam delete-role --role-name ${SERVICE_ROLE} --region ${REGION} || true - - rm -f ${SERVICE_ROLE_ARN_FILENAME} -fi - -echo "Listing thing principals..." -THING_PRINCIPAL=`aws iot list-thing-principals --region ${REGION} --thing-name ${VEHICLE_NAME} | jq -r ".principals[0]" || true` -echo "Detaching thing principal..." -aws iot detach-thing-principal --region ${REGION} --thing-name ${VEHICLE_NAME} --principal ${THING_PRINCIPAL} || true -echo "Detaching thing policy..." -aws iot detach-policy --region ${REGION} --policy-name ${THING_POLICY} --target ${THING_PRINCIPAL} || true -echo "Deleting thing..." -aws iot delete-thing --region ${REGION} --thing-name ${VEHICLE_NAME} || true -echo "Deleting thing policy..." -aws iot delete-policy --region ${REGION} --policy-name ${THING_POLICY} || true -CERTIFICATE_ID=`echo "${THING_PRINCIPAL}" | cut -d "/" -f 2 || true` -echo "Updating certificate as INACTIVE..." -aws iot update-certificate --region ${REGION} --certificate-id ${CERTIFICATE_ID} --new-status INACTIVE || true -echo "Deleting certificate..." -aws iot delete-certificate --region ${REGION} --certificate-id ${CERTIFICATE_ID} --force-delete || true - -rm -f ${VEHICLE_NAME_FILENAME} -rm -f ${THING_POLICY_FILENAME} -rm -f config/certificate.pem -rm -f config/private-key.key -rm -f config/creds.json -rm -f config/endpoint.txt -rm -f config/provisioning-qr-code.png - -echo "=========" -echo "Finished!" -echo "=========" diff --git a/tools/android-app/cloud/gen-aaos-vhal-info.py b/tools/android-app/cloud/gen-aaos-vhal-info.py index 22fc6c17..eec315a7 100644 --- a/tools/android-app/cloud/gen-aaos-vhal-info.py +++ b/tools/android-app/cloud/gen-aaos-vhal-info.py @@ -7,8 +7,6 @@ # Before running this script to generate the output files, firstly create a file called # aaos-vhal-types.h with the content from: # https://cs.android.com/android/platform/superproject/main/+/main:prebuilts/vndk/v30/x86/include/generated-headers/hardware/interfaces/automotive/vehicle/2.0/android.hardware.automotive.vehicle@2.0_genc++_headers/gen/android/hardware/automotive/vehicle/2.0/types.h -# The output in aaos-vhal-props-table.txt can then be copied to -# `tools/android-app/app/src/main/java/com/aws/iotfleetwise/AaosVehicleProperties.java` with open("aaos-vhal-types.h") as fp: text = fp.read() diff --git a/tools/android-app/cloud/network-interface-can-aaos-vhal.json b/tools/android-app/cloud/network-interface-can-aaos-vhal.json new file mode 100644 index 00000000..2869ccf3 --- /dev/null +++ b/tools/android-app/cloud/network-interface-can-aaos-vhal.json @@ -0,0 +1,11 @@ +[ + { + "interfaceId": "AAOS-VHAL-CAN", + "type": "CAN_INTERFACE", + "canInterface": { + "name": "can0", + "protocolName": "CAN", + "protocolVersion": "2.0B" + } + } +] diff --git a/tools/android-app/cloud/network-interface-can-external-gps.json b/tools/android-app/cloud/network-interface-can-external-gps.json new file mode 100644 index 00000000..16c57ec0 --- /dev/null +++ b/tools/android-app/cloud/network-interface-can-external-gps.json @@ -0,0 +1,11 @@ +[ + { + "interfaceId": "EXTERNAL-GPS-CAN", + "type": "CAN_INTERFACE", + "canInterface": { + "name": "can0", + "protocolName": "CAN", + "protocolVersion": "2.0B" + } + } +] diff --git a/tools/android-app/cloud/network-interfaces.json b/tools/android-app/cloud/network-interfaces.json deleted file mode 100644 index 5bb1fa64..00000000 --- a/tools/android-app/cloud/network-interfaces.json +++ /dev/null @@ -1,31 +0,0 @@ -[ - { - "interfaceId": "0", - "type": "OBD_INTERFACE", - "obdInterface": { - "name": "can0", - "requestMessageId": 2015, - "obdStandard": "J1979", - "pidRequestIntervalSeconds": 5, - "dtcRequestIntervalSeconds": 5 - } - }, - { - "interfaceId": "EXTERNAL-GPS-CAN", - "type": "CAN_INTERFACE", - "canInterface": { - "name": "can0", - "protocolName": "CAN", - "protocolVersion": "2.0B" - } - }, - { - "interfaceId": "AAOS-VHAL-CAN", - "type": "CAN_INTERFACE", - "canInterface": { - "name": "can0", - "protocolName": "CAN", - "protocolVersion": "2.0B" - } - } -] diff --git a/tools/android-app/cloud/setup-iotfleetwise.sh b/tools/android-app/cloud/setup-iotfleetwise.sh deleted file mode 100755 index 3c9a96a9..00000000 --- a/tools/android-app/cloud/setup-iotfleetwise.sh +++ /dev/null @@ -1,396 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -ENDPOINT_URL="" -ENDPOINT_URL_OPTION="" -REGION="us-east-1" -TIMESTAMP_FILENAME="config/timestamp.txt" -TIMESTAMP=`date +%s` -VEHICLE_NAME_FILENAME="config/vehicle-name.txt" -VEHICLE_NAME="" -SERVICE_ROLE="IoTFleetWiseServiceRole-${TIMESTAMP}" -DEFAULT_TIMESTREAM_DB_NAME="IoTFleetWiseDB-${TIMESTAMP}" -SERVICE_PRINCIPAL="iotfleetwise.amazonaws.com" -TIMESTREAM_DB_NAME_FILENAME="config/timestream-db-name.txt" -TIMESTREAM_DB_NAME="" -TIMESTREAM_TABLE_NAME="VehicleDataTable" -SERVICE_ROLE_ARN_FILENAME="config/service-role-arn.txt" -SERVICE_ROLE_ARN="" -CAMPAIGN_FILE="campaign-android.json" - -parse_args() { - while [ "$#" -gt 0 ]; do - case $1 in - --vehicle-name) - VEHICLE_NAME=$2 - shift - ;; - --timestream-db-name) - TIMESTREAM_DB_NAME=$2 - shift - ;; - --timestream-table-name) - TIMESTREAM_TABLE_NAME=$2 - shift - ;; - --service-role-arn) - SERVICE_ROLE_ARN=$2 - shift - ;; - --service-principal) - SERVICE_PRINCIPAL=$2 - shift - ;; - --endpoint-url) - ENDPOINT_URL=$2 - shift - ;; - --region) - REGION=$2 - shift - ;; - --help) - echo "Usage: $0 [OPTION]" - echo " --vehicle-name Vehicle name, by default will be read from ${VEHICLE_NAME_FILENAME}" - echo " --timestream-db-name Timestream database name, by default will be read from ${TIMESTREAM_DB_NAME_FILENAME}" - echo " --timestream-table-name Timetream table name, default: ${TIMESTREAM_TABLE_NAME}" - echo " --service-role-arn Service role ARN for accessing Timestream, by default will be read from ${SERVICE_ROLE_ARN_FILENAME}" - echo " --endpoint-url The endpoint URL used for AWS CLI calls" - echo " --region The region used for AWS CLI calls, default: ${REGION}" - echo " --service-principal AWS service principal for policies, default: ${SERVICE_PRINCIPAL}" - exit 0 - ;; - esac - shift - done - - if [ "${ENDPOINT_URL}" != "" ]; then - ENDPOINT_URL_OPTION="--endpoint-url ${ENDPOINT_URL}" - fi - - if [ "${VEHICLE_NAME}" == "" ]; then - if [ -f "${VEHICLE_NAME_FILENAME}" ]; then - VEHICLE_NAME=`cat ${VEHICLE_NAME_FILENAME}` - else - echo "Error: provision.sh script not run" - exit -1 - fi - fi - - if [ "${TIMESTREAM_DB_NAME}" == "" ]; then - if [ -f "${TIMESTREAM_DB_NAME_FILENAME}" ]; then - TIMESTREAM_DB_NAME=`cat ${TIMESTREAM_DB_NAME_FILENAME}` - fi - fi - - if [ "${SERVICE_ROLE_ARN}" == "" ]; then - if [ -f "${SERVICE_ROLE_ARN_FILENAME}" ]; then - SERVICE_ROLE_ARN=`cat ${SERVICE_ROLE_ARN_FILENAME}` - fi - fi -} - -echo "================================" -echo "AWS IoT FleetWise Android Script" -echo "================================" - -mkdir -p config -parse_args "$@" - -echo "Getting AWS account ID..." -ACCOUNT_ID=`aws sts get-caller-identity --query "Account" --output text` -echo ${ACCOUNT_ID} - -if [ "${TIMESTREAM_DB_NAME}" != "" ] && [ "${SERVICE_ROLE_ARN}" != "" ]; then - TIMESTREAM_TABLE_ARN="arn:aws:timestream:${REGION}:${ACCOUNT_ID}:database/${TIMESTREAM_DB_NAME}/table/${TIMESTREAM_TABLE_NAME}" -else - TIMESTREAM_DB_NAME=${DEFAULT_TIMESTREAM_DB_NAME} - - echo "Creating Timestream database..." - aws timestream-write create-database \ - --region ${REGION} \ - --database-name ${TIMESTREAM_DB_NAME} | jq -r .Database.Arn - - echo "Creating Timestream table..." - TIMESTREAM_TABLE_ARN=$( aws timestream-write create-table \ - --region ${REGION} \ - --database-name ${TIMESTREAM_DB_NAME} \ - --table-name ${TIMESTREAM_TABLE_NAME} \ - --retention-properties "{\"MemoryStoreRetentionPeriodInHours\":2, \ - \"MagneticStoreRetentionPeriodInDays\":2}" | jq -r .Table.Arn ) - - echo "Creating service role..." - SERVICE_ROLE_TRUST_POLICY=$(cat << EOF -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": [ - "$SERVICE_PRINCIPAL" - ] - }, - "Action": "sts:AssumeRole" - } - ] -} -EOF -) - SERVICE_ROLE_ARN=`aws iam create-role \ - --role-name "${SERVICE_ROLE}" \ - --assume-role-policy-document "${SERVICE_ROLE_TRUST_POLICY}" | jq -r .Role.Arn` - echo ${SERVICE_ROLE_ARN} - - echo "Waiting for role to be created..." - aws iam wait role-exists \ - --role-name "${SERVICE_ROLE}" - - echo "Creating service role policy..." - SERVICE_ROLE_POLICY=$(cat <<'EOF' -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "timestreamIngestion", - "Effect": "Allow", - "Action": [ - "timestream:WriteRecords", - "timestream:Select", - "timestream:DescribeTable" - ] - }, - { - "Sid": "timestreamDescribeEndpoint", - "Effect": "Allow", - "Action": [ - "timestream:DescribeEndpoints" - ], - "Resource": "*" - } - ] -} -EOF -) - SERVICE_ROLE_POLICY=`echo "${SERVICE_ROLE_POLICY}" \ - | jq ".Statement[0].Resource=\"arn:aws:timestream:${REGION}:${ACCOUNT_ID}:database/${TIMESTREAM_DB_NAME}/*\""` - SERVICE_ROLE_POLICY_ARN=`aws iam create-policy \ - --policy-name ${SERVICE_ROLE}-policy \ - --policy-document "${SERVICE_ROLE_POLICY}" | jq -r .Policy.Arn` - echo ${SERVICE_ROLE_POLICY_ARN} - - echo "Waiting for policy to be created..." - aws iam wait policy-exists \ - --policy-arn "${SERVICE_ROLE_POLICY_ARN}" - - echo "Attaching policy to service role..." - aws iam attach-role-policy \ - --policy-arn ${SERVICE_ROLE_POLICY_ARN} \ - --role-name "${SERVICE_ROLE}" - - echo ${TIMESTREAM_DB_NAME} > ${TIMESTREAM_DB_NAME_FILENAME} - echo ${SERVICE_ROLE_ARN} > ${SERVICE_ROLE_ARN_FILENAME} -fi - -NAME="${VEHICLE_NAME}-${TIMESTAMP}" -echo ${TIMESTAMP} > ${TIMESTAMP_FILENAME} - -echo "Deleting vehicle ${VEHICLE_NAME} if it already exists..." -aws iotfleetwise delete-vehicle \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --vehicle-name "${VEHICLE_NAME}" | jq -r .arn - -VEHICLE_NODE=`cat ../../cloud/vehicle-node.json` -OBD_NODES=`cat ../../cloud/obd-nodes.json` -DBC_NODES=`cat externalGpsNodes.json` -AAOS_NODES=`cat aaosVhalNodes.json` - -echo "Checking for existing signal catalog..." -SIGNAL_CATALOG_LIST=`aws iotfleetwise list-signal-catalogs \ - ${ENDPOINT_URL_OPTION} --region ${REGION}` -SIGNAL_CATALOG_COUNT=`echo ${SIGNAL_CATALOG_LIST} | jq '.summaries|length'` -# Currently only one signal catalog is supported by the service -if [ ${SIGNAL_CATALOG_COUNT} == 0 ]; then - SIGNAL_CATALOG_NAME="${NAME}-signal-catalog" - echo "No existing signal catalog" - echo "Creating signal catalog with Vehicle node..." - SIGNAL_CATALOG_ARN=`aws iotfleetwise create-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --nodes "${VEHICLE_NODE}" | jq -r .arn` - echo ${SIGNAL_CATALOG_ARN} -else - SIGNAL_CATALOG_NAME=`echo ${SIGNAL_CATALOG_LIST} | jq -r .summaries[0].name` - SIGNAL_CATALOG_ARN=`echo ${SIGNAL_CATALOG_LIST} | jq -r .summaries[0].arn` - - echo "Updating Vehicle node in signal catalog..." - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "Vehicle node" \ - --nodes-to-update "${VEHICLE_NODE}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} >&2 - exit -1 - else - echo "Node exists and is in use, continuing" - fi -fi - -echo "Updating OBD signals in signal catalog..." -if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "OBD signals" \ - --nodes-to-update "${OBD_NODES}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn -elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} >&2 - exit -1 -else - echo "Signals exist and are in use, continuing" -fi - -echo "Updating DBC signals in signal catalog..." -if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "DBC signals" \ - --nodes-to-update "${DBC_NODES}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn -elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} >&2 - exit -1 -else - echo "Signals exist and are in use, continuing" -fi - -echo "Updating AAOS signals in signal catalog..." -NODE_COUNT=`echo $AAOS_NODES | jq length` -for ((i=0; i<${NODE_COUNT}; i+=500)); do - NODES_SUBSET=`echo $AAOS_NODES | jq .[$i:$(($i+500))]` - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "AAOS signals" \ - --nodes-to-update "${NODES_SUBSET}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} >&2 - exit -1 - else - echo "Signals exist and are in use, continuing" - fi -done - -echo "Creating model manifest..." -# Make a list of all node names: -NODE_LIST=`( echo ${DBC_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\. ; \ - echo ${AAOS_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\. ; \ - echo ${OBD_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\. ) | jq -Rn [inputs]` -aws iotfleetwise create-model-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest \ - --signal-catalog-arn ${SIGNAL_CATALOG_ARN} \ - --nodes "${NODE_LIST}" | jq -r .arn - -echo "Activating model manifest..." -MODEL_MANIFEST_ARN=`aws iotfleetwise update-model-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest \ - --status ACTIVE | jq -r .arn` -echo ${MODEL_MANIFEST_ARN} - -echo "Creating decoder manifest with OBD signals..." -NETWORK_INTERFACES=`cat network-interfaces.json` -OBD_SIGNAL_DECODERS=`cat ../../cloud/obd-decoders.json` -DECODER_MANIFEST_ARN=`aws iotfleetwise create-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --model-manifest-arn ${MODEL_MANIFEST_ARN} \ - --network-interfaces "${NETWORK_INTERFACES}" \ - --signal-decoders "${OBD_SIGNAL_DECODERS}" | jq -r .arn` -echo ${DECODER_MANIFEST_ARN} - -echo "Updating decoder manifest with external GPS decoders..." -SIGNAL_DECODERS=`cat externalGpsDecoders.json` -aws iotfleetwise update-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --signal-decoders-to-add "${SIGNAL_DECODERS}" | jq -r .arn - -echo "Updating decoder manifest with AAOS VHAL decoders..." -SIGNAL_DECODERS=`cat aaosVhalDecoders.json` -DECODER_COUNT=`echo $SIGNAL_DECODERS | jq length` -for ((i=0; i<${DECODER_COUNT}; i+=200)); do - DECODER_SUBSET=`echo $SIGNAL_DECODERS | jq .[$i:$(($i+200))]` - aws iotfleetwise update-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --signal-decoders-to-add "${DECODER_SUBSET}" | jq -r .arn -done - -echo "Activating decoder manifest..." -aws iotfleetwise update-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --status ACTIVE | jq -r .arn - -echo "Creating vehicle ${VEHICLE_NAME}..." -aws iotfleetwise create-vehicle \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --decoder-manifest-arn ${DECODER_MANIFEST_ARN} \ - --association-behavior ValidateIotThingExists \ - --model-manifest-arn ${MODEL_MANIFEST_ARN} \ - --vehicle-name "${VEHICLE_NAME}" | jq -r .arn - -echo "Creating fleet..." -FLEET_ARN=`aws iotfleetwise create-fleet \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --fleet-id ${NAME}-fleet \ - --description "Description is required" \ - --signal-catalog-arn ${SIGNAL_CATALOG_ARN} | jq -r .arn` -echo ${FLEET_ARN} - -echo "Associating vehicle ${VEHICLE_NAME}..." -aws iotfleetwise associate-vehicle-fleet \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --fleet-id ${NAME}-fleet \ - --vehicle-name "${VEHICLE_NAME}" - -echo "Creating campaign from ${CAMPAIGN_FILE}..." -CAMPAIGN=`cat ${CAMPAIGN_FILE} \ - | jq .name=\"${NAME}-campaign\" \ - | jq .signalCatalogArn=\"${SIGNAL_CATALOG_ARN}\" \ - | jq .targetArn=\"${FLEET_ARN}\"` -aws iotfleetwise create-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --cli-input-json "${CAMPAIGN}" --data-destination-configs "[{\"timestreamConfig\":{\"timestreamTableArn\":\"${TIMESTREAM_TABLE_ARN}\",\"executionRoleArn\":\"${SERVICE_ROLE_ARN}\"}}]" | jq -r .arn - -echo "Waiting for campaign to become ready for approval..." -while true; do - sleep 5 - CAMPAIGN_STATUS=`aws iotfleetwise get-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign | jq -r .status` - if [ "${CAMPAIGN_STATUS}" == "WAITING_FOR_APPROVAL" ]; then - break - fi -done - -echo "Approving campaign..." -aws iotfleetwise update-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign \ - --action APPROVE | jq -r .arn - -echo -echo Finished, you can run the following SQL query to retrieve the collected data: -echo "--------------------------------" -echo "| Amazon Timestream SQL query: |" -echo "--------------------------------" -echo "SELECT * FROM \"${TIMESTREAM_DB_NAME}\".\"${TIMESTREAM_TABLE_NAME}\" WHERE vehicleName='${VEHICLE_NAME}' ORDER BY time DESC LIMIT 1000" diff --git a/tools/arm64-toolchain.cmake b/tools/arm64-toolchain.cmake index 774103c0..71916de8 100644 --- a/tools/arm64-toolchain.cmake +++ b/tools/arm64-toolchain.cmake @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR aarch64) set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) diff --git a/tools/armhf-toolchain.cmake b/tools/armhf-toolchain.cmake index e9672ffa..79048d3e 100644 --- a/tools/armhf-toolchain.cmake +++ b/tools/armhf-toolchain.cmake @@ -1,3 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_PROCESSOR arm) set(CMAKE_C_COMPILER arm-linux-gnueabihf-gcc) diff --git a/tools/build-fwe-cross-android.sh b/tools/build-fwe-cross-android.sh index 661640ed..b698ab02 100755 --- a/tools/build-fwe-cross-android.sh +++ b/tools/build-fwe-cross-android.sh @@ -36,9 +36,7 @@ parse_args() { parse_args "$@" -if [ -z "${FWE_ADDITIONAL_CMAKE_ARGS+x}" ]; then - FWE_ADDITIONAL_CMAKE_ARGS="" -fi +: ${FWE_ADDITIONAL_CMAKE_ARGS:=""} export PATH=/usr/local/android_sdk/cmake/${VERSION_CMAKE}/bin:${NATIVE_PREFIX}/bin:${PATH} mkdir -p build && cd build diff --git a/tools/build-fwe-cross-arm64.sh b/tools/build-fwe-cross-arm64.sh index c955fd48..363cb4db 100755 --- a/tools/build-fwe-cross-arm64.sh +++ b/tools/build-fwe-cross-arm64.sh @@ -4,17 +4,22 @@ set -eo pipefail +WITH_GREENGRASSV2_SUPPORT="false" WITH_ROS2_SUPPORT="false" parse_args() { while [ "$#" -gt 0 ]; do case $1 in + --with-greengrassv2-support) + WITH_GREENGRASSV2_SUPPORT="true" + ;; --with-ros2-support) WITH_ROS2_SUPPORT="true" ;; --help) echo "Usage: $0 [OPTION]" - echo " --with-ros2-support Build with ROS2 support" + echo " --with-greengrassv2-support Build with Greengrass V2 support" + echo " --with-ros2-support Build with ROS2 support" exit 0 ;; esac @@ -33,6 +38,9 @@ CMAKE_OPTIONS=" -DFWE_SECURITY_COMPILE_FLAGS=On -DCMAKE_TOOLCHAIN_FILE=/usr/local/aarch64-linux-gnu/lib/cmake/arm64-toolchain.cmake \ -DBUILD_TESTING=Off" +if ${WITH_GREENGRASSV2_SUPPORT}; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_GREENGRASSV2=On" +fi if ${WITH_ROS2_SUPPORT}; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_ROS2=On" fi diff --git a/tools/build-fwe-cross-armhf.sh b/tools/build-fwe-cross-armhf.sh index 6cc8dcfe..ecab32d2 100755 --- a/tools/build-fwe-cross-armhf.sh +++ b/tools/build-fwe-cross-armhf.sh @@ -4,12 +4,16 @@ set -eo pipefail +WITH_GREENGRASSV2_SUPPORT="false" WITH_IWAVE_GPS_SUPPORT="false" WITH_ROS2_SUPPORT="false" parse_args() { while [ "$#" -gt 0 ]; do case $1 in + --with-greengrassv2-support) + WITH_GREENGRASSV2_SUPPORT="true" + ;; --with-iwave-gps-support) WITH_IWAVE_GPS_SUPPORT="true" ;; @@ -18,8 +22,9 @@ parse_args() { ;; --help) echo "Usage: $0 [OPTION]" - echo " --with-ros2-support Build with ROS2 support" - echo " --with-iwave-gps-support Build with iWave GPS support" + echo " --with-greengrassv2-support Build with Greengrass V2 support" + echo " --with-iwave-gps-support Build with iWave GPS support" + echo " --with-ros2-support Build with ROS2 support" exit 0 ;; esac @@ -38,6 +43,9 @@ CMAKE_OPTIONS=" -DFWE_SECURITY_COMPILE_FLAGS=On -DCMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf/lib/cmake/armhf-toolchain.cmake \ -DBUILD_TESTING=Off" +if ${WITH_GREENGRASSV2_SUPPORT}; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_GREENGRASSV2=On" +fi if ${WITH_IWAVE_GPS_SUPPORT}; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_IWAVE_GPS=On" fi diff --git a/tools/build-fwe-native.sh b/tools/build-fwe-native.sh index 57f6b8a6..85c0795f 100755 --- a/tools/build-fwe-native.sh +++ b/tools/build-fwe-native.sh @@ -4,17 +4,22 @@ set -eo pipefail +WITH_GREENGRASSV2_SUPPORT="false" WITH_ROS2_SUPPORT="false" parse_args() { while [ "$#" -gt 0 ]; do case $1 in + --with-greengrassv2-support) + WITH_GREENGRASSV2_SUPPORT="true" + ;; --with-ros2-support) WITH_ROS2_SUPPORT="true" ;; --help) echo "Usage: $0 [OPTION]" - echo " --with-ros2-support Build with ROS2 support" + echo " --with-greengrassv2-support Build with Greengrass V2 support" + echo " --with-ros2-support Build with ROS2 support" exit 0 ;; esac @@ -33,6 +38,9 @@ CMAKE_OPTIONS=" -DFWE_SECURITY_COMPILE_FLAGS=On -DFWE_TEST_FAKETIME=On -DCMAKE_PREFIX_PATH=${PREFIX}" +if ${WITH_GREENGRASSV2_SUPPORT}; then + CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_GREENGRASSV2=On" +fi if ${WITH_ROS2_SUPPORT}; then CMAKE_OPTIONS="${CMAKE_OPTIONS} -DFWE_FEATURE_ROS2=On" fi diff --git a/tools/cansim/canigen.py b/tools/cansim/canigen.py index 20c3b393..8eb5c536 100755 --- a/tools/cansim/canigen.py +++ b/tools/cansim/canigen.py @@ -57,7 +57,7 @@ def __init__( if output_filename is not None: self._output_file = open(output_filename, "w") else: - self._can_bus = can.interface.Bus(self._interface, bustype="socketcan", fd=fd) + self._can_bus = can.interface.Bus(self._interface, interface="socketcan", fd=fd) self._obd_config = {} self._pid_names = [] self._dtc_names = [] @@ -271,7 +271,12 @@ def _obd_thread(self, ecu): tx = [0x7F, sid, 0x11] # NRC Service not supported # print(ecu['name']+' tx: '+str(tx)) if len(tx) > 1: - isotp_socket_phys.send(bytearray(tx)) + try: + isotp_socket_phys.send(bytearray(tx)) + except Exception: + pass # Ignore timeout errors + isotp_socket_phys.close() + isotp_socket_func.close() def get_sig_names(self): return self._sig_names diff --git a/tools/cansim/cansim.py b/tools/cansim/cansim.py index ec52bd96..f3d4fca3 100755 --- a/tools/cansim/cansim.py +++ b/tools/cansim/cansim.py @@ -20,8 +20,8 @@ database_filename=None if args.only_obd else "hscan.dbc", obd_config_filename="obd_config.json", ) -BRAKE_PRESSURE_SIGNAL = "BrakePedalPressure" -ENGINE_TORQUE_SIGNAL = "EngineTorque" +BRAKE_PRESSURE_SIGNAL = "DemoBrakePedalPressure" +ENGINE_TORQUE_SIGNAL = "DemoEngineTorque" def set_with_print(func, name, val): diff --git a/tools/cansim/hscan.dbc b/tools/cansim/hscan.dbc index 51d973b3..eb675efb 100644 --- a/tools/cansim/hscan.dbc +++ b/tools/cansim/hscan.dbc @@ -33,11 +33,17 @@ BS_: BU_: +BO_ 769 Connectivity: 1 Vector__XXX + SG_ NetworkType : 7|8@0+ (1,0) [0|3] "" Vector__XXX + BO_ 401 ECM: 8 Vector__XXX - SG_ EngineTorque : 3|12@0+ (0.5,-848) [-848|1199.5] "Nm" Vector__XXX + SG_ DemoEngineTorque : 3|12@0+ (0.5,-848) [-848|1199.5] "Nm" Vector__XXX BO_ 532 ABS: 6 Vector__XXX - SG_ BrakePedalPressure : 0|8@0+ (75,0) [0|19125] "kPa" Vector__XXX + SG_ DemoBrakePedalPressure : 0|8@0+ (75,0) [0|19125] "kPa" Vector__XXX + +BO_ 1217 BCM: 8 Vector__XXX + SG_ DemoEngineCoolantTemperature : 23|8@0+ (1,-40) [-40|215] "deg C" Vector__XXX BA_DEF_ SG_ "SignalType" STRING ; BA_DEF_ SG_ "SignalLongName" STRING ; @@ -45,5 +51,7 @@ BA_DEF_ BO_ "GenMsgCycleTime" INT 0 10000; BA_DEF_DEF_ "SignalType" ""; BA_DEF_DEF_ "SignalLongName" ""; BA_DEF_DEF_ "GenMsgCycleTime" 0; +BA_ "GenMsgCycleTime" BO_ 769 1000; BA_ "GenMsgCycleTime" BO_ 401 12; BA_ "GenMsgCycleTime" BO_ 532 50; +BA_ "GenMsgCycleTime" BO_ 1217 500; diff --git a/tools/cansim/obd_config.json b/tools/cansim/obd_config.json index 06f62b5d..ebf91675 100644 --- a/tools/cansim/obd_config.json +++ b/tools/cansim/obd_config.json @@ -2,7 +2,6 @@ "ecus": [ { "name": "engine", - "rx_ids": ["0x7DF", "0x7E0"], "tx_id": "0x7E8", "zero_padding": true, "pids": { @@ -37,7 +36,6 @@ }, { "name": "transmission", - "rx_ids": ["0x7DF", "0x7E1"], "tx_id": "0x7E9", "zero_padding": true, "pids": { diff --git a/tools/cfn-templates/fwdemo.yml b/tools/cfn-templates/fwdemo.yml index 90a9575d..9b06c7b3 100644 --- a/tools/cfn-templates/fwdemo.yml +++ b/tools/cfn-templates/fwdemo.yml @@ -58,6 +58,11 @@ Parameters: Description: "Maximum size of raw data buffer in bytes. Applicable when EnableROS2 is true." Type: Number Default: 2147483648 + FweLogLevel: + Description: "Set the log level in the FWE config file. Use 'Trace' for most verbosity." + Type: String + Default: "Info" + AllowedValues: ["Trace", "Info", "Warning", "Error"] Conditions: KeyPairSpecifiedCondition: !Not [!Equals [!Ref Ec2KeyPair, ""]] Resources: @@ -133,23 +138,13 @@ Resources: #!/bin/bash set -xeuo pipefail - # Wait for any existing package install to finish - i=0 - while true; do - if sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; then - i=0 - else - i=`expr $i + 1` - if expr $i \>= 10 > /dev/null; then - break - fi - fi - sleep 1 - done + # Disable unattended upgrades + systemctl stop unattended-upgrades + systemctl disable unattended-upgrades # Upgrade system and reboot if required - apt update - apt upgrade -y + apt update -o DPkg::Lock::Timeout=120 + apt upgrade -y -o DPkg::Lock::Timeout=120 if [ -f /var/run/reboot-required ]; then # Delete the UserData info file so that we run again after reboot rm -f /var/lib/cloud/instances/*/sem/config_scripts_user @@ -158,8 +153,8 @@ Resources: fi # Install helper scripts: - apt update - apt install -y python3-setuptools + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 python3-setuptools mkdir -p /opt/aws/bin wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz @@ -179,8 +174,8 @@ Resources: printf "RateLimitBurst=0\nSystemMaxUse=1G\n" >> /etc/systemd/journald.conf # Install packages - apt update - apt install -y wget ec2-instance-connect htop jq unzip + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 wget ec2-instance-connect htop jq unzip # Install AWS CLI: curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" @@ -231,6 +226,8 @@ Resources: ./tools/configure-fwe.sh \ --input-config-file "configuration/static-config.json" \ --output-config-file "/etc/aws-iot-fleetwise/config-0.json" \ + --log-color Yes \ + --log-level ${FweLogLevel} \ --vehicle-name "${AWS::StackName}" \ --endpoint-url "${IoTThing.iotEndpoint}" \ --topic-prefix '${IoTMqttTopicPrefix}' \ @@ -266,6 +263,10 @@ Resources: # Signal init complete: /opt/aws/bin/cfn-signal --stack ${AWS::StackName} --resource Ec2Instance --region ${AWS::Region} + + # Re-enable unattended upgrades + systemctl enable unattended-upgrades + systemctl start unattended-upgrades Ec2Instance: Type: AWS::EC2::Instance CreationPolicy: diff --git a/tools/cfn-templates/fwdev.yml b/tools/cfn-templates/fwdev.yml index f8a2c1d0..afb8457a 100644 --- a/tools/cfn-templates/fwdev.yml +++ b/tools/cfn-templates/fwdev.yml @@ -117,23 +117,13 @@ Resources: #!/bin/bash set -euo pipefail - # Wait for any existing package install to finish - i=0 - while true; do - if sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; then - i=0 - else - i=`expr $i + 1` - if expr $i \>= 10 > /dev/null; then - break - fi - fi - sleep 1 - done + # Disable unattended upgrades + systemctl stop unattended-upgrades + systemctl disable unattended-upgrades # Upgrade system and reboot if required - apt update - apt upgrade -y + apt update -o DPkg::Lock::Timeout=120 + apt upgrade -y -o DPkg::Lock::Timeout=120 if [ -f /var/run/reboot-required ]; then # Delete the UserData info file so that we run again after reboot rm -f /var/lib/cloud/instances/*/sem/config_scripts_user @@ -142,8 +132,8 @@ Resources: fi # Install helper scripts: - apt update - apt install -y python3-setuptools + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 python3-setuptools mkdir -p /opt/aws/bin wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz @@ -156,8 +146,8 @@ Resources: trap error_handler ERR # Install packages - apt update - apt install -y ec2-instance-connect htop jq unzip zip + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 ec2-instance-connect htop jq unzip zip # Install AWS CLI: curl "https://awscli.amazonaws.com/awscli-exe-linux-${Arch}.zip" -o "awscliv2.zip" @@ -167,6 +157,10 @@ Resources: # Signal init complete: /opt/aws/bin/cfn-signal --stack ${AWS::StackName} --resource Ec2Instance --region ${AWS::Region} + + # Re-enable unattended upgrades + systemctl enable unattended-upgrades + systemctl start unattended-upgrades - Arch: !FindInMap [InstanceArchMap, !Ref Ec2InstanceType, Arch] Ec2Instance: Type: AWS::EC2::Instance diff --git a/tools/cfn-templates/vision-system-data-jupyter.yml b/tools/cfn-templates/vision-system-data-jupyter.yml index 49423010..cebc5517 100644 --- a/tools/cfn-templates/vision-system-data-jupyter.yml +++ b/tools/cfn-templates/vision-system-data-jupyter.yml @@ -340,22 +340,14 @@ Resources: - | #!/bin/bash set -euovx pipefail - # Wait for any existing package install to finish - i=0 - while true; do - if sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; then - i=0 - else - i=`expr $i + 1` - if expr $i \>= 10 > /dev/null; then - break - fi - fi - sleep 1 - done + + # Disable unattended upgrades + systemctl stop unattended-upgrades + systemctl disable unattended-upgrades + # Upgrade system and reboot if required - apt update - apt upgrade -y + apt update -o DPkg::Lock::Timeout=120 + apt upgrade -y -o DPkg::Lock::Timeout=120 if [ -f /var/run/reboot-required ]; then # Delete the UserData info file so that we run again after reboot rm -f /var/lib/cloud/instances/*/sem/config_scripts_user @@ -364,8 +356,8 @@ Resources: fi # Install helper scripts: configsets=OnCreate - apt update - apt install -y python3-setuptools python3-pip git markdown + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 python3-setuptools python3-pip git markdown mkdir -p /opt/aws/bin wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-latest.tar.gz python3 -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-py3-latest.tar.gz @@ -384,8 +376,8 @@ Resources: } trap error_handler ERR # Install packages - apt update - apt install -y ec2-instance-connect htop jq unzip zip + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 ec2-instance-connect htop jq unzip zip # Install AWS CLI: curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip" unzip -q awscliv2.zip diff --git a/tools/cloud/.gitignore b/tools/cloud/.gitignore index c8398cd6..6e25c27c 100644 --- a/tools/cloud/.gitignore +++ b/tools/cloud/.gitignore @@ -3,3 +3,5 @@ demo.env collected-data-* ros2-nodes.json ros2-decoders.json +can-nodes.json +can-decoders.json diff --git a/tools/cloud/clean-up.sh b/tools/cloud/clean-up.sh index f0a2847d..c5601fbd 100755 --- a/tools/cloud/clean-up.sh +++ b/tools/cloud/clean-up.sh @@ -15,6 +15,7 @@ SERVICE_ROLE="IoTFleetWiseServiceRole" SERVICE_ROLE_POLICY_ARN="arn:aws:iam::${ACCOUNT_ID}:policy/" FLEET_SIZE=1 BATCH_SIZE=$((`nproc`*4)) +CAMPAIGN_NAMES="" if [ -f demo.env ]; then source demo.env @@ -68,7 +69,7 @@ parse_args() { fi if [ "${VEHICLE_NAME}" == "" ]; then echo "Error: Vehicle name not provided" - exit -1 + exit 1 fi if [ "${DISAMBIGUATOR}" != "" ]; then DISAMBIGUATOR="-${DISAMBIGUATOR}" @@ -99,25 +100,30 @@ echo "Vehicle name: ${VEHICLE_NAME}" echo "Fleet size: ${FLEET_SIZE}" echo "Vehicles: ${VEHICLES[@]}" +FAILED_CLEAN_UP_STEPS="" + # $1 is the campaign name suspend_and_delete_campaign(){ - echo "Suspending campaign..." + echo "Suspending campaign $1..." aws iotfleetwise update-campaign \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ --name $1 \ - --action SUSPEND 2> /dev/null | jq -r .arn || true + --action SUSPEND | jq -r .arn \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} suspend-campaign" - echo "Deleting campaign..." + echo "Deleting campaign $1..." aws iotfleetwise delete-campaign \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name $1 2> /dev/null | jq -r .arn || true + --name $1 | jq -r .arn \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-campaign" } -suspend_and_delete_campaign ${NAME}-campaign -suspend_and_delete_campaign ${NAME}-campaign-s3-json -suspend_and_delete_campaign ${NAME}-campaign-s3-parquet +for CAMPAIGN_NAME in $(echo $CAMPAIGN_NAMES | tr "," " "); do + suspend_and_delete_campaign $CAMPAIGN_NAME +done for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do + PIDS=() for ((j=0; j<${BATCH_SIZE} && i+j<${FLEET_SIZE}; j++)); do vehicle=${VEHICLES[$((i+j))]} echo "Disassociating vehicle ${vehicle}..." @@ -127,14 +133,18 @@ for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do aws iotfleetwise disassociate-vehicle-fleet \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ --fleet-id ${NAME}-fleet \ - --vehicle-name "${vehicle}" 2> /dev/null || true \ - 2>&3 &} 3>&2 2>/dev/null + --vehicle-name "${vehicle}" \ + 2>&3 &} 3>&2 + PIDS+=($!) done # Wait for all background processes to finish - wait + for PID in ${PIDS[@]}; do + wait $PID || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} disassociate-vehicle-fleet" + done done for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do + PIDS=() for ((j=0; j<${BATCH_SIZE} && i+j<${FLEET_SIZE}; j++)); do vehicle=${VEHICLES[$((i+j))]} echo "Deleting vehicle ${vehicle}..." @@ -143,36 +153,105 @@ for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do { \ aws iotfleetwise delete-vehicle \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --vehicle-name "${vehicle}" 2> /dev/null || true \ - 2>&3 &} 3>&2 2>/dev/null + --vehicle-name "${vehicle}" \ + 2>&3 &} 3>&2 + PIDS+=($!) done # Wait for all background processes to finish - wait + for PID in ${PIDS[@]}; do + wait $PID || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-vehicle" + done done echo "Deleting fleet..." aws iotfleetwise delete-fleet \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --fleet-id ${NAME}-fleet 2> /dev/null || true + --fleet-id ${NAME}-fleet \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-fleet" echo "Deleting decoder manifest..." aws iotfleetwise delete-decoder-manifest \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest 2> /dev/null || true + --name ${NAME}-decoder-manifest \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-decoder-manifest" echo "Deleting model manifest..." aws iotfleetwise delete-model-manifest \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest 2> /dev/null || true - -echo "Deleting service role and policy..." -aws iam detach-role-policy --role-name ${SERVICE_ROLE} --policy-arn ${SERVICE_ROLE_POLICY_ARN} --region ${REGION} || true -aws iam delete-policy --policy-arn ${SERVICE_ROLE_POLICY_ARN} --region ${REGION} || true -aws iam delete-role --role-name ${SERVICE_ROLE} --region ${REGION} || true + --name ${NAME}-model-manifest \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-model-manifest" if [ "${SIGNAL_CATALOG}" != "" ]; then echo "Deleting signal catalog..." aws iotfleetwise delete-signal-catalog \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG} 2> /dev/null || true + --name ${SIGNAL_CATALOG} \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-signal-catalog" +fi + +# The role might not exist depending on which type of campaign was created, so only try to delete +# it if it exists. +echo "Checking if role exists: ${SERVICE_ROLE}" +if ! GET_ROLE_OUTPUT=$(aws iam get-role --region ${REGION} --role-name ${SERVICE_ROLE} 2>&1); then + if ! echo ${GET_ROLE_OUTPUT} | grep -q "NoSuchEntity"; then + echo ${GET_ROLE_OUTPUT} + FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} get-role" + fi +else + echo "Deleting service role and policy..." + ATTACHED_POLICIES=$( + aws iam list-attached-role-policies \ + --region ${REGION} \ + --role-name ${SERVICE_ROLE} --query 'AttachedPolicies[].PolicyArn' --output text + ) + for POLICY_ARN in $ATTACHED_POLICIES; do + echo "Detaching policy: $POLICY_ARN from role: $SERVICE_ROLE" + aws iam detach-role-policy \ + --region ${REGION} \ + --role-name ${SERVICE_ROLE} --policy-arn ${POLICY_ARN} \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} detach-role-policy" + + MAX_ATTEMPTS=60 + for i in $(seq 1 $MAX_ATTEMPTS); do + echo "Deleting policy ${POLICY_ARN}. This might take a while until detach-role-policy operation propagates" + if aws iam delete-policy --region ${REGION} --policy-arn ${POLICY_ARN}; then + break + elif $i -eq $MAX_ATTEMPTS; then + FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-policy" + else + sleep 1 + fi + done + done + + INLINE_POLICIES=$( + aws iam list-role-policies \ + --region ${REGION} \ + --role-name ${SERVICE_ROLE} --query 'PolicyNames[]' --output text + ) + for POLICY_NAME in $INLINE_POLICIES; do + echo "Deleting inline policy: $POLICY_NAME from role: $SERVICE_ROLE" + aws iam delete-role-policy \ + --region ${REGION} \ + --role-name ${SERVICE_ROLE} --policy-name ${POLICY_NAME} \ + || FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-role-policy" + done + + + MAX_ATTEMPTS=60 + for i in $(seq 1 $MAX_ATTEMPTS); do + echo "Deleting service role ${SERVICE_ROLE}. This may take a while until the policy deletion propagates." + if aws iam delete-role --region ${REGION} --role-name ${SERVICE_ROLE}; then + break + elif $i -eq $MAX_ATTEMPTS; then + FAILED_CLEAN_UP_STEPS="${FAILED_CLEAN_UP_STEPS} delete-role" + else + sleep 1 + fi + done +fi + +if [ "${FAILED_CLEAN_UP_STEPS}" != "" ]; then + echo "Failed to clean up the following resources: ${FAILED_CLEAN_UP_STEPS}" + exit 1 fi diff --git a/tools/cloud/demo.sh b/tools/cloud/demo.sh index e2efa93f..1fcd9d87 100755 --- a/tools/cloud/demo.sh +++ b/tools/cloud/demo.sh @@ -4,6 +4,7 @@ set -euo pipefail +SCRIPT_DIR="$(dirname "$(realpath "$0")")" ACCOUNT_ID=`aws sts get-caller-identity --query "Account" --output text` UUID="$(cat /proc/sys/kernel/random/uuid)" # A short string to avoid resource name conflicts. Since most resources names have a small @@ -16,25 +17,20 @@ ENDPOINT_URL_OPTION="" REGION="us-east-1" DEFAULT_VEHICLE_NAME="fwdemo" VEHICLE_NAME="" -TIMESTREAM_DB_NAME="IoTFleetWiseDB-${DISAMBIGUATOR}" +TIMESTREAM_DB_NAME="" TIMESTREAM_TABLE_NAME="VehicleDataTable" SERVICE_ROLE="IoTFleetWiseServiceRole" SERVICE_ROLE_POLICY_ARN="" SERVICE_PRINCIPAL="iotfleetwise.amazonaws.com" BUCKET_NAME="" SKIP_S3_POLICY=true -CAMPAIGN_FILE="" -DEFAULT_CAMPAIGN_FILE="campaign-brake-event.json" -S3_CAMPAIGN_FILE="" -DEFAULT_S3_CAMPAIGN_FILE=${DEFAULT_CAMPAIGN_FILE} -DEFAULT_S3_CAMPAIGN_FILE_VISION_SYSTEM_DATA="campaign-brake-event-vision-system-data.json" -DBC_FILE="" -DEFAULT_DBC_FILE="hscan.dbc" -ROS2_ENABLED=false -ROS2_CONFIG_FILE="" -DEFAULT_ROS2_CONFIG_FILE="ros2-config.json" +CAMPAIGN_FILES=() +NODE_FILES=() +DECODER_FILES=() +NETWORK_INTERFACE_FILES=() +S3_FORMAT="JSON" +SKIP_ACCOUNT_REGISTRATION=false CLEAN_UP=false -S3_UPLOAD=false FLEET_SIZE=1 BATCH_SIZE=$((`nproc`*4)) HEALTH_CHECK_RETRIES=360 # About 30mins @@ -43,17 +39,10 @@ FORCE_REGISTRATION=false MIN_CLI_VERSION="aws-cli/2.13.39" CREATED_SIGNAL_CATALOG_NAME="" CREATED_S3_BUCKET="" -INCLUDED_SIGNALS="" -DEFAULT_INCLUDED_SIGNALS=" - Vehicle.ECM.DemoEngineTorque, - Vehicle.ABS.DemoBrakePedalPressure," -DEFAULT_INCLUDED_SIGNALS_VISION_SYSTEM_DATA=" - Vehicle.Acceleration.linear_acceleration.x, - Vehicle.Acceleration.linear_acceleration.y, - Vehicle.Speed.data, - Vehicle.Acceleration.angular_velocity.z, - Vehicle.Cameras.Front.Image.data," -EXCLUDED_SIGNALS="" +CREATED_CAMPAIGN_NAMES="" +INCLUDE_SIGNALS="" +EXCLUDE_SIGNALS="" +DATA_DESTINATION="TIMESTREAM" parse_args() { while [ "$#" -gt 0 ]; do @@ -69,29 +58,28 @@ parse_args() { --clean-up) CLEAN_UP=true ;; - --enable-s3-upload) - S3_UPLOAD=true + --data-destination) + DATA_DESTINATION=$2 + shift ;; --campaign-file) - CAMPAIGN_FILE=$2 + CAMPAIGN_FILES+=("$2") shift ;; - --s3-campaign-file) - S3_CAMPAIGN_FILE=$2 + --s3-format) + S3_FORMAT=$2 shift ;; - --dbc-file) - DBC_FILE=$2 + --node-file) + NODE_FILES+=("$2") shift ;; - --enable-ros2) - ROS2_ENABLED=true - S3_UPLOAD=true + --decoder-file) + DECODER_FILES+=("$2") + shift ;; - --ros2-config-file) - ROS2_ENABLED=true - S3_UPLOAD=true - ROS2_CONFIG_FILE=$2 + --network-interface-file) + NETWORK_INTERFACE_FILES+=("$2") shift ;; --bucket-name) @@ -101,6 +89,9 @@ parse_args() { --set-bucket-policy) SKIP_S3_POLICY=false ;; + --skip-account-registration) + SKIP_ACCOUNT_REGISTRATION=true + ;; --service-principal) SERVICE_PRINCIPAL=$2 shift @@ -113,12 +104,12 @@ parse_args() { REGION=$2 shift ;; - --included-signals) - INCLUDED_SIGNALS="$2" + --include-signals) + INCLUDE_SIGNALS="$2" shift ;; - --excluded-signals) - EXCLUDED_SIGNALS="$2" + --exclude-signals) + EXCLUDE_SIGNALS="$2" shift ;; --help) @@ -127,18 +118,18 @@ parse_args() { echo " --fleet-size Size of fleet, default: ${FLEET_SIZE}. When greater than 1," echo " the instance number will be appended to each" echo " Vehicle name after a '-', e.g. ${DEFAULT_VEHICLE_NAME}-42" - echo " --campaign-file Campaign JSON file, default: ${DEFAULT_CAMPAIGN_FILE}" - echo " --s3-campaign-file Campaign JSON file for S3 collection, default: ${DEFAULT_S3_CAMPAIGN_FILE}, or" - echo " ${DEFAULT_S3_CAMPAIGN_FILE_VISION_SYSTEM_DATA} with --enable-ros2." - echo " --dbc-file DBC file, default: ${DEFAULT_DBC_FILE}" + echo " --node-file Node JSON file. Can be used multiple times for multiple files." + echo " --decoder-file Decoder JSON file. Can be used multiple times for multiple files." + echo " --network-interface-file Network interface JSON file. Can be used multiple times for multiple files." + echo " --campaign-file Campaign JSON file. Can be used multiple times for multiple files." + echo " --data-destination Data destination, either TIMESTREAM or S3, default: ${DATA_DESTINATION}" + echo " --s3-format Either JSON or PARQUET, default: ${S3_FORMAT}" echo " --bucket-name S3 bucket name, if not specified a new bucket will be created" echo " --set-bucket-policy Sets the required bucket policy" echo " --clean-up Delete created resources" - echo " --enable-s3-upload Create campaigns to upload data to S3" - echo " --enable-ros2 Enables ROS2 collection. Implies --enable-s3-upload." - echo " --ros2-config-file ROS2 config file, default: ${DEFAULT_ROS2_CONFIG_FILE}. Implies --enable-ros2." - echo " --included-signals Comma separated list of signals to include in HTML plot" - echo " --excluded-signals Comma separated list of signals to exclude from HTML plot" + echo " --skip-account-registration Don't check account registration nor try to register it. Most features should work without registration." + echo " --include-signals Comma separated list of signals to include in HTML plot" + echo " --exclude-signals Comma separated list of signals to exclude from HTML plot" echo " --endpoint-url The endpoint URL used for AWS CLI calls" echo " --service-principal AWS service principal for policies, default: ${SERVICE_PRINCIPAL}" echo " --region The region used for AWS CLI calls, default: ${REGION}" @@ -171,62 +162,6 @@ if [ "${VEHICLE_NAME}" == "" ]; then fi fi -if [ "${INCLUDED_SIGNALS}" == "" ]; then - if [ "${DBC_FILE}" == "" ] && [ "${CAMPAIGN_FILE}" == "" ]; then - INCLUDED_SIGNALS="${DEFAULT_INCLUDED_SIGNALS}" - fi - if [ "${ROS2_CONFIG_FILE}" == "" ] && [ "${S3_CAMPAIGN_FILE}" == "" ]; then - INCLUDED_SIGNALS="${INCLUDED_SIGNALS}${DEFAULT_INCLUDED_SIGNALS_VISION_SYSTEM_DATA}" - fi -fi - -if ( [ "${DBC_FILE}" != "" ] && [ "${CAMPAIGN_FILE}" == "" ] ); then - echo -n "Enter campaign file name: " - read CAMPAIGN_FILE - if [ "${CAMPAIGN_FILE}" == "" ]; then - echo "Error: Please provide campaign file name for custom DBC file" >&2 - exit -1 - fi -fi -if ( [ "${ROS2_CONFIG_FILE}" != "" ] && [ "${S3_CAMPAIGN_FILE}" == "" ] ); then - echo -n "Enter S3 campaign file name: " - read S3_CAMPAIGN_FILE - if [ "${S3_CAMPAIGN_FILE}" == "" ]; then - echo "Error: Please provide campaign file name for ROS2 config file" >&2 - exit -1 - fi -fi - -if ${ROS2_ENABLED} && [ "${BUCKET_NAME}" == "" ]; then - echo -n "Enter the name of an existing S3 bucket created in ${REGION}: " - read BUCKET_NAME - if [ "${BUCKET_NAME}" == "" ]; then - echo "Error: Please provide an existing S3 bucket" >&2 - exit -1 - fi -fi - -if [ "${CAMPAIGN_FILE}" == "" ]; then - CAMPAIGN_FILE=${DEFAULT_CAMPAIGN_FILE} -fi -if [ "${S3_CAMPAIGN_FILE}" == "" ]; then - if ${ROS2_ENABLED}; then - S3_CAMPAIGN_FILE=${DEFAULT_S3_CAMPAIGN_FILE_VISION_SYSTEM_DATA} - else - S3_CAMPAIGN_FILE=${DEFAULT_S3_CAMPAIGN_FILE} - fi -fi - -if [ "${ROS2_CONFIG_FILE}" == "" ]; then - ROS2_CONFIG_FILE="${DEFAULT_ROS2_CONFIG_FILE}" -fi - -if [ ${S3_UPLOAD} == true ] && [ "${BUCKET_NAME}" == "" ]; then - CREATED_S3_BUCKET="iot-fleetwise-demo-${S3_SUFFIX}" - BUCKET_NAME=${CREATED_S3_BUCKET} - SKIP_S3_POLICY=false -fi - if [ ${FLEET_SIZE} -gt 1 ]; then VEHICLES=( $(seq -f "${VEHICLE_NAME}-%g" 0 $((${FLEET_SIZE}-1))) ) else @@ -240,7 +175,7 @@ echo -n "Date: " date +%Y-%m-%dT%H:%M:%S%z echo "Disambiguator: ${DISAMBIGUATOR}" echo "Vehicle name: ${VEHICLE_NAME}" -echo "Fleet Size: ${FLEET_SIZE}" +echo "Fleet size: ${FLEET_SIZE}" echo "Vehicles: ${VEHICLES[@]}" # AWS CLI v1.x has a double base64 encoding issue @@ -253,12 +188,6 @@ if [[ "${CLI_VERSION}" < "${MIN_CLI_VERSION}" ]]; then exit -1 fi -if ${ROS2_ENABLED} && ! ( aws iotfleetwise create-decoder-manifest help | grep -q ros2 ); then - echo "Error: The AWS CLI does not yet support the vision system data feature." - echo " Please update your AWS CLI and/or try again later." - exit -1 -fi - save_variables() { # Export some variables to a .env file so that they can be referenced by other scripts echo " @@ -270,7 +199,13 @@ REGION=${REGION} SIGNAL_CATALOG=${CREATED_SIGNAL_CATALOG_NAME} TIMESTREAM_DB_NAME=${TIMESTREAM_DB_NAME} TIMESTREAM_TABLE_NAME=${TIMESTREAM_TABLE_NAME} -BUCKET_NAME=${CREATED_S3_BUCKET} +CREATED_S3_BUCKET=${CREATED_S3_BUCKET} +CAMPAIGN_NAMES=${CREATED_CAMPAIGN_NAMES} +BUCKET_NAME=${BUCKET_NAME} +DATA_DESTINATION=${DATA_DESTINATION} +S3_SUFFIX=${S3_SUFFIX} +INCLUDE_SIGNALS=\"${INCLUDE_SIGNALS}\" +EXCLUDE_SIGNALS=\"${EXCLUDE_SIGNALS}\" " > demo.env } @@ -313,130 +248,79 @@ approve_campaign() { --action APPROVE | jq -r .arn } +# $1 is the vehicle name , $2 is the expected campaign name +check_vehicle_healthy() { + for ((k=0; k<${HEALTH_CHECK_RETRIES}; k++)); do + VEHICLE_STATUS=`aws iotfleetwise get-vehicle-status \ + ${ENDPOINT_URL_OPTION} --region ${REGION} \ + --vehicle-name "$1"` + for ((l=0; ; l++)); do + CAMPAIGN_NAME=`echo ${VEHICLE_STATUS} | jq -r .campaigns[${l}].campaignName` + CAMPAIGN_STATUS=`echo ${VEHICLE_STATUS} | jq -r .campaigns[${l}].status` + # If the campaign was not found (when the index is out-of-range jq will return 'null') + if [ "${CAMPAIGN_NAME}" == "null" ]; then + echo "Error: Campaign not found in vehicle status for vehicle $1" >&2 + exit -1 + # If the campaign was found + elif [ "${CAMPAIGN_NAME}" == "$2" ]; then + if [ "${CAMPAIGN_STATUS}" == "HEALTHY" ]; then + break 2 + fi + break + fi + done + sleep 5 + done + if ((k>=HEALTH_CHECK_RETRIES)); then + echo "Error: Health check timeout for vehicle $1" >&2 + exit -1 + fi +} + trap cleanup EXIT echo "Getting AWS account ID..." echo ${ACCOUNT_ID} -echo "Getting account registration status..." -if REGISTER_ACCOUNT_STATUS=`get_account_status 2>&1`; then - ACCOUNT_STATUS=`echo "${REGISTER_ACCOUNT_STATUS}" | jq -r .accountStatus` -elif ! echo ${REGISTER_ACCOUNT_STATUS} | grep -q "ResourceNotFoundException"; then - echo "${REGISTER_ACCOUNT_STATUS}" >&2 - exit -1 -else - ACCOUNT_STATUS="NOT_REGISTERED" -fi -echo ${ACCOUNT_STATUS} -if [ "${ACCOUNT_STATUS}" == "REGISTRATION_SUCCESS" ]; then - echo "Account is already registered" -elif [ "${ACCOUNT_STATUS}" == "REGISTRATION_PENDING" ]; then - echo "Waiting for account to be registered..." -else - register_account -fi - -# Due to IAM's eventual consistency, the registration may initially fail because the -# recently created role and the attached policies take some time to propagate. -REGISTRATION_ATTEMPTS=0 -while [ "${ACCOUNT_STATUS}" != "REGISTRATION_SUCCESS" ]; do - sleep 5 - REGISTER_ACCOUNT_STATUS=`get_account_status` - ACCOUNT_STATUS=`echo "${REGISTER_ACCOUNT_STATUS}" | jq -r .accountStatus` - if [ "${ACCOUNT_STATUS}" == "REGISTRATION_FAILURE" ]; then - echo "Error: Registration failed" >&2 - ((REGISTRATION_ATTEMPTS+=1)) - if ((REGISTRATION_ATTEMPTS >= MAX_ATTEMPTS_ON_REGISTRATION_FAILURE)); then - echo "${REGISTER_ACCOUNT_STATUS}" >&2 - echo "All ${MAX_ATTEMPTS_ON_REGISTRATION_FAILURE} registration attempts failed" >&2 - exit -1 - else - register_account - fi +if ! ${SKIP_ACCOUNT_REGISTRATION}; then + echo "Getting account registration status..." + if REGISTER_ACCOUNT_STATUS=`get_account_status 2>&1`; then + ACCOUNT_STATUS=`echo "${REGISTER_ACCOUNT_STATUS}" | jq -r .accountStatus` + elif ! echo ${REGISTER_ACCOUNT_STATUS} | grep -q "ResourceNotFoundException"; then + echo "${REGISTER_ACCOUNT_STATUS}" >&2 + exit -1 + else + ACCOUNT_STATUS="NOT_REGISTERED" + fi + echo ${ACCOUNT_STATUS} + if [ "${ACCOUNT_STATUS}" == "REGISTRATION_SUCCESS" ]; then + echo "Account is already registered" + elif [ "${ACCOUNT_STATUS}" == "REGISTRATION_PENDING" ]; then + echo "Waiting for account to be registered..." + else + register_account fi -done - -echo "Creating Timestream database..." -aws timestream-write create-database \ - --region ${REGION} \ - --database-name ${TIMESTREAM_DB_NAME} | jq -r .Database.Arn - -echo "Creating Timestream table..." -TIMESTREAM_TABLE_ARN=$( aws timestream-write create-table \ - --region ${REGION} \ - --database-name ${TIMESTREAM_DB_NAME} \ - --table-name ${TIMESTREAM_TABLE_NAME} \ - --retention-properties "{\"MemoryStoreRetentionPeriodInHours\":2, \ - \"MagneticStoreRetentionPeriodInDays\":2}" | jq -r .Table.Arn ) - -echo "Creating service role..." -SERVICE_ROLE_TRUST_POLICY=$(cat << EOF -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": [ - "$SERVICE_PRINCIPAL" - ] - }, - "Action": "sts:AssumeRole" - } - ] -} -EOF -) -SERVICE_ROLE_ARN=`aws iam create-role \ - --role-name "${SERVICE_ROLE}" \ - --assume-role-policy-document "${SERVICE_ROLE_TRUST_POLICY}" | jq -r .Role.Arn` -echo ${SERVICE_ROLE_ARN} - -echo "Waiting for role to be created..." -aws iam wait role-exists \ - --role-name "${SERVICE_ROLE}" -echo "Creating service role policy..." -SERVICE_ROLE_POLICY=$(cat <<'EOF' -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "timestreamIngestion", - "Effect": "Allow", - "Action": [ - "timestream:WriteRecords", - "timestream:Select", - "timestream:DescribeTable" - ] - }, - { - "Sid": "timestreamDescribeEndpoint", - "Effect": "Allow", - "Action": [ - "timestream:DescribeEndpoints" - ], - "Resource": "*" - } - ] -} -EOF -) -SERVICE_ROLE_POLICY=`echo "${SERVICE_ROLE_POLICY}" \ - | jq ".Statement[0].Resource=\"arn:aws:timestream:${REGION}:${ACCOUNT_ID}:database/${TIMESTREAM_DB_NAME}/*\""` -SERVICE_ROLE_POLICY_ARN=`aws iam create-policy \ - --policy-name ${SERVICE_ROLE}-policy \ - --policy-document "${SERVICE_ROLE_POLICY}" | jq -r .Policy.Arn` -echo ${SERVICE_ROLE_POLICY_ARN} - -echo "Waiting for policy to be created..." -aws iam wait policy-exists \ - --policy-arn "${SERVICE_ROLE_POLICY_ARN}" - -echo "Attaching policy to service role..." -aws iam attach-role-policy \ - --policy-arn ${SERVICE_ROLE_POLICY_ARN} \ - --role-name "${SERVICE_ROLE}" + # Due to IAM's eventual consistency, the registration may initially fail because the + # recently created role and the attached policies take some time to propagate. + REGISTRATION_ATTEMPTS=0 + while [ "${ACCOUNT_STATUS}" != "REGISTRATION_SUCCESS" ]; do + sleep 5 + REGISTER_ACCOUNT_STATUS=`get_account_status` + ACCOUNT_STATUS=`echo "${REGISTER_ACCOUNT_STATUS}" | jq -r .accountStatus` + if [ "${ACCOUNT_STATUS}" == "REGISTRATION_FAILURE" ]; then + echo "Error: Registration failed" >&2 + ((REGISTRATION_ATTEMPTS+=1)) + if ((REGISTRATION_ATTEMPTS >= MAX_ATTEMPTS_ON_REGISTRATION_FAILURE)); then + echo "${REGISTER_ACCOUNT_STATUS}" >&2 + echo "All ${MAX_ATTEMPTS_ON_REGISTRATION_FAILURE} registration attempts failed" >&2 + exit -1 + else + register_account + fi + fi + done +fi for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do for ((j=0; j<${BATCH_SIZE} && i+j<${FLEET_SIZE}; j++)); do @@ -454,21 +338,26 @@ for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do wait done -if [ ${S3_UPLOAD} == true ]; then +if [ ${#CAMPAIGN_FILES[@]} -eq 0 ]; then + echo "No campaign file(s) provided, so no data destination to setup" +elif [ "${DATA_DESTINATION}" == "S3" ]; then echo "S3 upload is enabled" + if [ "${BUCKET_NAME}" == "" ]; then + CREATED_S3_BUCKET="iot-fleetwise-demo-${S3_SUFFIX}" + BUCKET_NAME=${CREATED_S3_BUCKET} + SKIP_S3_POLICY=false + fi echo "Checking if S3 bucket exists..." BUCKET_LIST=$( aws s3 ls ) if grep -q "$BUCKET_NAME" <<< "$BUCKET_LIST"; then echo "S3 bucket already exists" - elif ${ROS2_ENABLED}; then - echo "Error: S3 bucket '${BUCKET_NAME}' does not exist" >&2 - exit -1 else echo "Creating S3 bucket..." aws s3 mb s3://$BUCKET_NAME --region $REGION + SKIP_S3_POLICY=false fi - if [ ${SKIP_S3_POLICY} == false ]; then + if ! ${SKIP_S3_POLICY}; then echo "Adding S3 bucket policy..." cat << EOF > s3-bucket-policy.json { @@ -512,66 +401,94 @@ EOF echo "Error: ACLs are enabled for bucket ${BUCKET_NAME}. Disable them at https://s3.console.aws.amazon.com/s3/bucket/${BUCKET_NAME}/property/oo/edit" exit -1 fi -fi +elif [ "${DATA_DESTINATION}" == "TIMESTREAM" ]; then # Timestream + TIMESTREAM_DB_NAME="IoTFleetWiseDB-${DISAMBIGUATOR}" + echo "Creating Timestream database..." + aws timestream-write create-database \ + --region ${REGION} \ + --database-name ${TIMESTREAM_DB_NAME} | jq -r .Database.Arn -VEHICLE_NODE=`cat vehicle-node.json` -OBD_NODES=`cat obd-nodes.json` -if [ "${DBC_FILE}" == "" ]; then - DBC_NODES=`python3 dbc-to-nodes.py ${DEFAULT_DBC_FILE}` -else - DBC_NODES=`python3 dbc-to-nodes.py ${DBC_FILE}` -fi -if ! ${ROS2_ENABLED}; then - ROS2_NODES="[]" + echo "Creating Timestream table..." + TIMESTREAM_TABLE_ARN=$( aws timestream-write create-table \ + --region ${REGION} \ + --database-name ${TIMESTREAM_DB_NAME} \ + --table-name ${TIMESTREAM_TABLE_NAME} \ + --retention-properties "{\"MemoryStoreRetentionPeriodInHours\":2, \ + \"MagneticStoreRetentionPeriodInDays\":2}" | jq -r .Table.Arn ) + + echo "Creating service role..." + SERVICE_ROLE_TRUST_POLICY=$(cat << EOF +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "$SERVICE_PRINCIPAL" + ] + }, + "Action": "sts:AssumeRole" + } +] +} +EOF +) + SERVICE_ROLE_ARN=`aws iam create-role \ + --role-name "${SERVICE_ROLE}" \ + --assume-role-policy-document "${SERVICE_ROLE_TRUST_POLICY}" | jq -r .Role.Arn` + echo ${SERVICE_ROLE_ARN} + + echo "Waiting for role to be created..." + aws iam wait role-exists \ + --role-name "${SERVICE_ROLE}" + + echo "Creating service role policy..." + SERVICE_ROLE_POLICY=$(cat <<'EOF' +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "timestreamIngestion", + "Effect": "Allow", + "Action": [ + "timestream:WriteRecords", + "timestream:Select", + "timestream:DescribeTable" + ] + }, + { + "Sid": "timestreamDescribeEndpoint", + "Effect": "Allow", + "Action": [ + "timestream:DescribeEndpoints" + ], + "Resource": "*" + } + ] +} +EOF +) + SERVICE_ROLE_POLICY=`echo "${SERVICE_ROLE_POLICY}" \ + | jq ".Statement[0].Resource=\"arn:aws:timestream:${REGION}:${ACCOUNT_ID}:database/${TIMESTREAM_DB_NAME}/*\""` + aws iam put-role-policy \ + --role-name "${SERVICE_ROLE}" \ + --policy-name ${SERVICE_ROLE}-policy \ + --policy-document "${SERVICE_ROLE_POLICY}" else - python3 ros2-to-nodes.py --config ${ROS2_CONFIG_FILE} --output ros2-nodes.json - ROS2_NODES=`cat ros2-nodes.json` + echo "Error: Unknown data destination ${DATA_DESTINATION}" + exit -1 fi +VEHICLE_NODE=`cat ${SCRIPT_DIR}/vehicle-node.json` if SIGNAL_CATALOG_ARN=`aws iotfleetwise create-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-signal-catalog \ - --nodes "${VEHICLE_NODE}" 2>/dev/null | jq -r .arn`; then + ${ENDPOINT_URL_OPTION} --region ${REGION} \ + --name ${NAME}-signal-catalog \ + --nodes "${VEHICLE_NODE}" 2>/dev/null | jq -r .arn`; then echo "Created new signal catalog: ${SIGNAL_CATALOG_ARN}" - CREATED_SIGNAL_CATALOG_NAME="${NAME}-signal-catalog" - - echo "Adding OBD signals to signal catalog..." - aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-signal-catalog \ - --description "OBD signals" \ - --nodes-to-add "${OBD_NODES}" | jq -r .arn - - echo "Adding DBC signals to signal catalog..." - aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-signal-catalog \ - --description "DBC signals" \ - --nodes-to-add "${DBC_NODES}" | jq -r .arn - - if ${ROS2_ENABLED}; then - echo "Adding ROS2 signals to signal catalog..." - aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-signal-catalog \ - --description "ROS2 signals" \ - --nodes-to-update "${ROS2_NODES}" | jq -r .arn - fi - - echo "Add an attribute to signal catalog..." - aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-signal-catalog \ - --description "DBC Attributes" \ - --nodes-to-add '[{ - "attribute": { - "dataType": "STRING", - "description": "Color", - "fullyQualifiedName": "Vehicle.Color", - "defaultValue":"Red" - }} - ]' | jq -r .arn + SIGNAL_CATALOG_NAME="${NAME}-signal-catalog" + CREATED_SIGNAL_CATALOG_NAME="${SIGNAL_CATALOG_NAME}" else echo "Checking for existing signal catalogs..." SIGNAL_CATALOG_LIST=`aws iotfleetwise list-signal-catalogs \ @@ -579,163 +496,96 @@ else SIGNAL_CATALOG_NAME=`echo ${SIGNAL_CATALOG_LIST} | jq -r .summaries[0].name` SIGNAL_CATALOG_ARN=`echo ${SIGNAL_CATALOG_LIST} | jq -r .summaries[0].arn` echo "Reusing existing signal catalog: ${SIGNAL_CATALOG_ARN}" +fi - echo "Updating Vehicle node in signal catalog..." - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "Vehicle node" \ - --nodes-to-update "${VEHICLE_NODE}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo "${UPDATE_SIGNAL_CATALOG_STATUS}" >&2 - exit -1 - else - echo "Node exists and is in use, continuing" - fi - - echo "Updating OBD signals in signal catalog..." - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "OBD signals" \ - --nodes-to-update "${OBD_NODES}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo "${UPDATE_SIGNAL_CATALOG_STATUS}" >&2 - exit -1 - else - echo "Signals exist and are in use, continuing" - fi - - echo "Updating DBC signals in signal catalog..." - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "DBC signals" \ - --nodes-to-update "${DBC_NODES}" 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo "${UPDATE_SIGNAL_CATALOG_STATUS}" >&2 - exit -1 - else - echo "Signals exist and are in use, continuing" - fi - - if ${ROS2_ENABLED}; then - echo "Updating ROS2 signals in signal catalog..." +if [ ${#NODE_FILES[@]} -eq 0 ]; then + echo "No node file(s) provided, so finishing now" + exit 0 +fi +ALL_NODES="[]" +for NODE_FILE in "${NODE_FILES[@]}"; do + echo "Updating nodes in signal catalog from ${NODE_FILE}" + NODES=`cat ${NODE_FILE}` + ALL_NODES=`echo "${ALL_NODES}" | jq ".+=${NODES}"` + NODE_COUNT=`echo $NODES | jq length` + for ((i=0; i<${NODE_COUNT}; i+=500)); do + NODES_SUBSET=`echo $NODES | jq .[$i:$(($i+500))]` if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ --name ${SIGNAL_CATALOG_NAME} \ - --description "ROS2 signals" \ - --nodes-to-update "${ROS2_NODES}" 2>&1`; then + --nodes-to-update "${NODES_SUBSET}" 2>&1`; then echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo "${UPDATE_SIGNAL_CATALOG_STATUS}" >&2 + echo ${UPDATE_SIGNAL_CATALOG_STATUS} >&2 exit -1 else echo "Signals exist and are in use, continuing" fi - fi - - echo "Updating color attribute" - if UPDATE_SIGNAL_CATALOG_STATUS=`aws iotfleetwise update-signal-catalog \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${SIGNAL_CATALOG_NAME} \ - --description "DBC Attributes" \ - --nodes-to-update '[{ - "attribute": { - "dataType": "STRING", - "description": "Color", - "fullyQualifiedName": "Vehicle.Color", - "defaultValue":"Red" - }} - ]' 2>&1`; then - echo ${UPDATE_SIGNAL_CATALOG_STATUS} | jq -r .arn - elif ! echo ${UPDATE_SIGNAL_CATALOG_STATUS} | grep -q "InvalidSignalsException"; then - echo "${UPDATE_SIGNAL_CATALOG_STATUS}" >&2 - exit -1 - else - echo "Signals exist and are in use, continuing" - fi -fi + done +done echo "Creating model manifest..." # Make a list of all node names: -NODE_LIST=`echo ${DBC_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\. ; \ - echo ${OBD_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\.` -if ${ROS2_ENABLED}; then - NODE_LIST=`echo "${NODE_LIST}" ; \ - echo ${ROS2_NODES} | jq -r .[].sensor.fullyQualifiedName | grep Vehicle\\.` -fi -NODE_LIST=`echo "${NODE_LIST}" | jq -Rn [inputs]` +NODE_LIST=`echo "${ALL_NODES}" | jq -r ".[] | .actuator,.sensor | .fullyQualifiedName" | grep Vehicle\\. | jq -Rn [inputs]` aws iotfleetwise create-model-manifest \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ --name ${NAME}-model-manifest \ --signal-catalog-arn ${SIGNAL_CATALOG_ARN} \ --nodes "${NODE_LIST}" | jq -r .arn -echo "Updating attribute in model manifest..." -MODEL_MANIFEST_ARN=`aws iotfleetwise update-model-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest \ - --nodes-to-add 'Vehicle.Color' | jq -r .arn` -echo ${MODEL_MANIFEST_ARN} - echo "Activating model manifest..." -MODEL_MANIFEST_ARN=`aws iotfleetwise update-model-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-model-manifest \ - --status ACTIVE | jq -r .arn` -echo ${MODEL_MANIFEST_ARN} +while true; do + if UPDATE_MODEL_MANIFEST_STATUS=`aws iotfleetwise update-model-manifest \ + ${ENDPOINT_URL_OPTION} --region ${REGION} \ + --name ${NAME}-model-manifest \ + --status ACTIVE`; then + MODEL_MANIFEST_ARN=`echo ${UPDATE_MODEL_MANIFEST_STATUS} | jq -r .arn` + echo ${MODEL_MANIFEST_ARN} + break + elif ! echo ${UPDATE_MODEL_MANIFEST_STATUS} | grep -q "ConflictException"; then + echo "${UPDATE_MODEL_MANIFEST_STATUS}" >&2 + exit -1 + else + DELAY=5 + echo "Signal catalog in use, waiting for ${DELAY} seconds and retrying..." + sleep ${DELAY} + fi +done -NETWORK_INTERFACES=`cat network-interfaces.json` -if ${ROS2_ENABLED}; then - ROS2_NETWORK_INTERFACE=`cat network-interface-ros2.json` - NETWORK_INTERFACES=`echo ${NETWORK_INTERFACES} | jq -r ". += ${ROS2_NETWORK_INTERFACE}"` +if [ ${#NETWORK_INTERFACE_FILES[@]} -eq 0 ]; then + echo "No network interface file(s) provided, so finishing now" + exit 0 fi - -echo "Creating decoder manifest with OBD signals..." -OBD_SIGNAL_DECODERS=`cat obd-decoders.json` +NETWORK_INTERFACES="[]" +for NETWORK_INTERFACE_FILE in "${NETWORK_INTERFACE_FILES[@]}"; do + echo "Reading network interfaces from ${NETWORK_INTERFACE_FILE}" + NEXT_NETWORK_INTERFACE=`cat ${NETWORK_INTERFACE_FILE}` + NETWORK_INTERFACES=`echo "${NETWORK_INTERFACES}" | jq ".+=${NEXT_NETWORK_INTERFACE}"` +done +echo "Creating decoder manifest with network interfaces..." DECODER_MANIFEST_ARN=`aws iotfleetwise create-decoder-manifest \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ --name ${NAME}-decoder-manifest \ --model-manifest-arn ${MODEL_MANIFEST_ARN} \ - --network-interfaces "${NETWORK_INTERFACES}" \ - --signal-decoders "${OBD_SIGNAL_DECODERS}" | jq -r .arn` + --network-interfaces "${NETWORK_INTERFACES}" | jq -r .arn` echo ${DECODER_MANIFEST_ARN} -echo "Adding DBC signals to decoder manifest..." -if [ "${DBC_FILE}" == "" ]; then - DBC=`cat ${DEFAULT_DBC_FILE} | base64 -w0` - # Make map of node name to DBC signal name, i.e. {"Vehicle.SignalName":"SignalName"...} - NODE_TO_DBC_MAP=`echo ${DBC_NODES} | jq '.[].sensor.fullyQualifiedName//""|match("Vehicle\\\\.\\\\w+\\\\.(.+)")|{(.captures[0].string):.string}'|jq -s add` - NETWORK_FILE_DEFINITIONS=`echo [] \ - | jq .[0].canDbc.signalsMap="${NODE_TO_DBC_MAP}" \ - | jq .[0].canDbc.networkInterface="\"1\"" \ - | jq .[0].canDbc.canDbcFiles[0]="\"${DBC}\""` - aws iotfleetwise import-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --network-file-definitions "${NETWORK_FILE_DEFINITIONS}" | jq -r .arn -else - SIGNAL_DECODERS=`python3 dbc-to-decoders.py ${DBC_FILE}` - aws iotfleetwise update-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --signal-decoders-to-add "${SIGNAL_DECODERS}" | jq -r .arn -fi - -if ${ROS2_ENABLED}; then - echo "Adding ROS2 signals to decoder manifest..." - python3 ros2-to-decoders.py --config ${ROS2_CONFIG_FILE} --output ros2-decoders.json - ROS2_DECODERS=`cat ros2-decoders.json` - DECODER_MANIFEST_ARN=`aws iotfleetwise update-decoder-manifest \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-decoder-manifest \ - --signal-decoders-to-add "${ROS2_DECODERS}" | jq -r .arn` - echo ${DECODER_MANIFEST_ARN} +if [ ${#DECODER_FILES[@]} -eq 0 ]; then + echo "No decoder file(s) provided, so finishing now" + exit 0 fi +for DECODER_FILE in "${DECODER_FILES[@]}"; do + echo "Updating decoders in decoder manifest from ${DECODER_FILE}..." + DECODERS=`cat ${DECODER_FILE}` + DECODER_COUNT=`echo $DECODERS | jq length` + for ((i=0; i<${DECODER_COUNT}; i+=200)); do + DECODER_SUBSET=`echo $DECODERS | jq .[$i:$(($i+200))]` + aws iotfleetwise update-decoder-manifest \ + ${ENDPOINT_URL_OPTION} --region ${REGION} \ + --name ${NAME}-decoder-manifest \ + --signal-decoders-to-add "${DECODER_SUBSET}" | jq -r .arn + done +done echo "Activating decoder manifest..." aws iotfleetwise update-decoder-manifest \ @@ -767,7 +617,6 @@ for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do --decoder-manifest-arn ${DECODER_MANIFEST_ARN} \ --association-behavior ValidateIotThingExists \ --model-manifest-arn ${MODEL_MANIFEST_ARN} \ - --attributes '{"Vehicle.Color":"Red"}' \ --vehicle-name "${VEHICLE}" >/dev/null \ 2>&3 &} 3>&2 2>/dev/null done @@ -800,206 +649,160 @@ for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do wait done -echo "Creating campaign from ${CAMPAIGN_FILE}..." -CAMPAIGN=`cat ${CAMPAIGN_FILE} \ - | jq .name=\"${NAME}-campaign\" \ - | jq .signalCatalogArn=\"${SIGNAL_CATALOG_ARN}\" \ - | jq .targetArn=\"${FLEET_ARN}\"` -aws iotfleetwise create-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --cli-input-json "${CAMPAIGN}" --data-destination-configs "[{\"timestreamConfig\":{\"timestreamTableArn\":\"${TIMESTREAM_TABLE_ARN}\",\"executionRoleArn\":\"${SERVICE_ROLE_ARN}\"}}]"| jq -r .arn - -approve_campaign ${NAME}-campaign - -if [ ${S3_UPLOAD} == true ]; then - echo "Creating campaign from ${S3_CAMPAIGN_FILE} for S3..." - CAMPAIGN=`cat ${S3_CAMPAIGN_FILE} \ - | jq .name=\"${NAME}-campaign-s3-json\" \ - | jq .signalCatalogArn=\"${SIGNAL_CATALOG_ARN}\" \ - | jq .targetArn=\"${FLEET_ARN}\"` - aws iotfleetwise create-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --cli-input-json "${CAMPAIGN}" --data-destination-configs "[{\"s3Config\":{\"bucketArn\":\"arn:aws:s3:::${BUCKET_NAME}\",\"prefix\":\"${NAME}-campaign-s3-${S3_SUFFIX}\",\"dataFormat\":\"JSON\",\"storageCompressionFormat\":\"NONE\"}}]"| jq -r .arn - - approve_campaign ${NAME}-campaign-s3-json - - echo "Creating campaign from ${S3_CAMPAIGN_FILE} for S3..." - CAMPAIGN=`cat ${S3_CAMPAIGN_FILE} \ - | jq .name=\"${NAME}-campaign-s3-parquet\" \ - | jq .signalCatalogArn=\"${SIGNAL_CATALOG_ARN}\" \ - | jq .targetArn=\"${FLEET_ARN}\"` - aws iotfleetwise create-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --cli-input-json "${CAMPAIGN}" --data-destination-configs "[{\"s3Config\":{\"bucketArn\":\"arn:aws:s3:::${BUCKET_NAME}\",\"prefix\":\"${NAME}-campaign-s3-${S3_SUFFIX}\",\"dataFormat\":\"PARQUET\",\"storageCompressionFormat\":\"NONE\"}}]"| jq -r .arn - - approve_campaign ${NAME}-campaign-s3-parquet -fi - -# The following two actions(Suspending, Resuming) are only for demo purpose, it won't affect the campaign status -sleep 2 -echo "Suspending campaign..." -aws iotfleetwise update-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign \ - --action SUSPEND | jq -r .arn - -sleep 2 -echo "Resuming campaign..." -aws iotfleetwise update-campaign \ - ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --name ${NAME}-campaign \ - --action RESUME | jq -r .arn - -check_vehicle_healthy() { - for ((k=0; k<${HEALTH_CHECK_RETRIES}; k++)); do - VEHICLE_STATUS=`aws iotfleetwise get-vehicle-status \ +CAMPAIGN_COUNTER=0 +if [ ${#CAMPAIGN_FILES[@]} -eq 0 ]; then + echo "No campaign file(s) provided, so finishing now." +else + for CAMPAIGN_FILE in "${CAMPAIGN_FILES[@]}"; do + CAMPAIGN_COUNTER=$((${CAMPAIGN_COUNTER}+1)) + CAMPAIGN_NAME=${NAME}-campaign-${CAMPAIGN_COUNTER} + echo "Creating campaign from ${CAMPAIGN_FILE} as ${CAMPAIGN_NAME}" + CAMPAIGN=`cat ${CAMPAIGN_FILE} \ + | jq .name=\"${CAMPAIGN_NAME}\" \ + | jq .signalCatalogArn=\"${SIGNAL_CATALOG_ARN}\" \ + | jq .targetArn=\"${FLEET_ARN}\"` + if [ "${DATA_DESTINATION}" == "S3" ]; then + echo "(Data destination is S3 with format ${S3_FORMAT})" + CAMPAIGN=`echo "${CAMPAIGN}" \ + | jq ".dataDestinationConfigs=[{\"s3Config\":{\"bucketArn\":\"arn:aws:s3:::${BUCKET_NAME}\",\"prefix\":\"iot-fleetwise-demo-${DISAMBIGUATOR}-s3-${S3_SUFFIX}\",\"dataFormat\":\"${S3_FORMAT}\",\"storageCompressionFormat\":\"NONE\"}}]"` + elif [ "${DATA_DESTINATION}" == "TIMESTREAM" ]; then + CAMPAIGN=`echo "${CAMPAIGN}" \ + | jq ".dataDestinationConfigs=[{\"timestreamConfig\":{\"timestreamTableArn\":\"${TIMESTREAM_TABLE_ARN}\",\"executionRoleArn\":\"${SERVICE_ROLE_ARN}\"}}]"` + else + echo "Error: Unknown data destination ${DATA_DESTINATION}" + exit -1 + fi + aws iotfleetwise create-campaign \ ${ENDPOINT_URL_OPTION} --region ${REGION} \ - --vehicle-name "$1"` - for ((l=0; ; l++)); do - CAMPAIGN_NAME=`echo ${VEHICLE_STATUS} | jq -r .campaigns[${l}].campaignName` - CAMPAIGN_STATUS=`echo ${VEHICLE_STATUS} | jq -r .campaigns[${l}].status` - # If the campaign was not found (when the index is out-of-range jq will return 'null') - if [ "${CAMPAIGN_NAME}" == "null" ]; then - echo "Error: Campaign not found in vehicle status for vehicle $1" >&2 - exit -1 - # If the campaign was found \ - elif [ "${CAMPAIGN_NAME}" == "${NAME}-campaign" ]; then - if [ "${CAMPAIGN_STATUS}" == "HEALTHY" ]; then - break 2 - fi - break - fi + --cli-input-json "${CAMPAIGN}" | jq -r .arn + if (( CAMPAIGN_COUNTER > 1 )); then + CREATED_CAMPAIGN_NAMES+="," + fi + CREATED_CAMPAIGN_NAMES+=$CAMPAIGN_NAME + + approve_campaign ${CAMPAIGN_NAME} + + for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do + for ((j=0; j<${BATCH_SIZE} && i+j<${FLEET_SIZE}; j++)); do + VEHICLE=${VEHICLES[$((i+j))]} + echo "Waiting until the status of ${CAMPAIGN_NAME} on vehicle ${VEHICLE} is healthy..." + # This output group is run in a background process. Note that stderr is redirected to stream 3 and back, + # to print stderr from the output group, but not info about the background process. + { \ + check_vehicle_healthy "${VEHICLE}" "${CAMPAIGN_NAME}"\ + 2>&3 &} 3>&2 2>/dev/null + done + # Wait for all background processes to finish + wait done - sleep 5 - done - if ((k>=HEALTH_CHECK_RETRIES)); then - echo "Error: Health check timeout for vehicle $1" >&2 - exit -1 - fi -} - -for ((i=0; i<${FLEET_SIZE}; i+=${BATCH_SIZE})); do - for ((j=0; j<${BATCH_SIZE} && i+j<${FLEET_SIZE}; j++)); do - VEHICLE=${VEHICLES[$((i+j))]} - echo "Waiting until status of vehicle ${VEHICLE} is healthy..." - # This output group is run in a background process. Note that stderr is redirected to stream 3 and back, - # to print stderr from the output group, but not info about the background process. - { \ - check_vehicle_healthy "${VEHICLE}" \ - 2>&3 &} 3>&2 2>/dev/null done - # Wait for all background processes to finish - wait -done - -DELAY=30 -echo "Waiting ${DELAY} seconds for data to be collected..." -sleep ${DELAY} -echo "The DB Name is ${TIMESTREAM_DB_NAME}" -echo "The DB Table is ${TIMESTREAM_TABLE_NAME}" + COLLECTED_DATA_DIR="collected-data-${DISAMBIGUATOR}/" + mkdir -p ${COLLECTED_DATA_DIR} -COLLECTED_DATA_DIR="collected-data-${DISAMBIGUATOR}/" -mkdir -p ${COLLECTED_DATA_DIR} - -for VEHICLE in ${VEHICLES[@]}; do - echo "Querying Timestream for vehicle ${VEHICLE}..." - aws timestream-query query \ - --region ${REGION} \ - --query-string "SELECT * FROM \"${TIMESTREAM_DB_NAME}\".\"${TIMESTREAM_TABLE_NAME}\" \ - WHERE vehicleName = '${VEHICLE}' \ - AND time between ago(1m) and now() ORDER BY time ASC" \ - > ${COLLECTED_DATA_DIR}${VEHICLE}-timestream-result.json -done - -if [ "${DBC_FILE}" != "" ]; then - echo "----------------------------------------------------------------------------------------" - echo "| Note: A custom DBC file was used. If an error occurs below due to no collected data, |" - echo "| you need to modify cansim.py or run FWE with a real vehicle. |" - echo "----------------------------------------------------------------------------------------" -fi - -OUTPUT_FILES=() -for VEHICLE in ${VEHICLES[@]}; do - echo "Converting from Timestream JSON to HTML..." - OUTPUT_FILE_HTML="${COLLECTED_DATA_DIR}${VEHICLE}.html" - OUTPUT_FILES+=(${OUTPUT_FILE_HTML}) - python3 timestream-to-html.py \ - --vehicle-name ${VEHICLE} \ - --files ${COLLECTED_DATA_DIR}${VEHICLE}-timestream-result.json \ - --html-filename ${OUTPUT_FILE_HTML} \ - --include-signals "${INCLUDED_SIGNALS}" \ - --exclude-signals "${EXCLUDED_SIGNALS}" -done + if [ "${DATA_DESTINATION}" == "S3" ]; then + DELAY=1200 + SLEEP_TIME=300 + echo "Waiting `expr ${DELAY} / 60` minutes for data to be collected and uploaded to S3..." + for ((i=DELAY; i>0; i-=SLEEP_TIME)); do + sleep $((i > SLEEP_TIME ? SLEEP_TIME : i)) + echo "..." # Print something to prevent the terminal timing out + done -echo "You can now view the collected data." -echo "----------------------------------" -echo "| Collected data in HTML format: |" -echo "----------------------------------" -for FILE in ${OUTPUT_FILES[@]}; do - echo `realpath ${FILE}` -done + echo "Downloading files from S3..." + S3_URL="s3://${BUCKET_NAME}/iot-fleetwise-demo-${DISAMBIGUATOR}-s3-${S3_SUFFIX}/processed-data/" + if ! COLLECTED_FILES=`aws s3 ls --recursive ${S3_URL}`; then + echo "Error: no collected data was found at ${S3_URL}" + exit -1 + fi + echo "${COLLECTED_FILES}" | while read LINE; do + KEY=`echo ${LINE} | cut -d ' ' -f4` + aws s3 cp s3://${BUCKET_NAME}/${KEY} ${COLLECTED_DATA_DIR} + done -if ${S3_UPLOAD}; then - DELAY=1200 - echo "Waiting 20 minutes for data to be collected and uploaded to S3..." - sleep ${DELAY} + for VEHICLE in ${VEHICLES[@]}; do + echo "Converting from Firehose ${S3_FORMAT} to HTML..." + OUTPUT_FILE_HTML="${COLLECTED_DATA_DIR}${VEHICLE}.html" + OUTPUT_FILE_S3_LINKS="${COLLECTED_DATA_DIR}${VEHICLE}-s3-links.txt" + python3 ${SCRIPT_DIR}/firehose-to-html.py \ + --vehicle-name ${VEHICLE} \ + --files ${COLLECTED_DATA_DIR}part-*.* \ + --html-filename ${OUTPUT_FILE_HTML} \ + --s3-links-filename ${OUTPUT_FILE_S3_LINKS} \ + --include-signals "${INCLUDE_SIGNALS}" \ + --exclude-signals "${EXCLUDE_SIGNALS}" + + if [ -s ${OUTPUT_FILE_S3_LINKS} ]; then + echo "Downloading the first 10 linked files..." + i=0 + cat ${OUTPUT_FILE_S3_LINKS} | while read LINE; do + if ((i < 10)); then + IMAGE_FILE=`basename ${LINE}.jpg` + # Remove random prefix so that filenames begin with date + IMAGE_FILE=`echo ${IMAGE_FILE} | sed -E 's/^[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}\-//'` + aws s3 cp ${LINE} ${COLLECTED_DATA_DIR}${IMAGE_FILE} + fi + i=$((i+1)) + done + fi + done - echo "Downloading files from S3..." - S3_URL="s3://${BUCKET_NAME}/${NAME}-campaign-s3-${S3_SUFFIX}/processed-data/" - if ! COLLECTED_FILES=`aws s3 ls --recursive ${S3_URL}`; then - echo "Error: no collected data was found at ${S3_URL}" - exit -1 - fi - echo "${COLLECTED_FILES}" | while read LINE; do - KEY=`echo ${LINE} | cut -d ' ' -f4` - aws s3 cp s3://${BUCKET_NAME}/${KEY} ${COLLECTED_DATA_DIR} - done + echo "Zipping up collected data..." + zip ${COLLECTED_DATA_DIR}${NAME}.zip ${COLLECTED_DATA_DIR}* + + echo "You can now view the collected data." + echo "----------------------------------------------------" + echo "| Collected S3 data for all campaigns for ${NAME}: |" + echo "----------------------------------------------------" + echo `realpath ${COLLECTED_DATA_DIR}${NAME}.zip` + elif [ "${DATA_DESTINATION}" == "TIMESTREAM" ]; then # Data destination is Timestream + DELAY=30 + echo "Waiting ${DELAY} seconds for data to be collected..." + sleep ${DELAY} + + echo "The DB Name is ${TIMESTREAM_DB_NAME}" + echo "The DB Table is ${TIMESTREAM_TABLE_NAME}" + + OUTPUT_FILES=() + + for VEHICLE in ${VEHICLES[@]}; do + SQL_QUERY="SELECT * FROM \"${TIMESTREAM_DB_NAME}\".\"${TIMESTREAM_TABLE_NAME}\" WHERE vehicleName='${VEHICLE}' AND time between ago(1m) and now() ORDER BY time ASC" + echo "Querying Timestream for vehicle ${VEHICLE}..." + echo "${SQL_QUERY}" + TIMESTREAM_JSON_FILE="${COLLECTED_DATA_DIR}${VEHICLE}-timestream-result.json" + if aws timestream-query query \ + --region ${REGION} \ + --query-string "${SQL_QUERY}" \ + > ${TIMESTREAM_JSON_FILE}; then + echo "Saved timestream results to ${TIMESTREAM_JSON_FILE}" + OUTPUT_FILE_HTML="${COLLECTED_DATA_DIR}${VEHICLE}.html" + OUTPUT_FILES+=(${OUTPUT_FILE_HTML}) + echo "Converting from Timestream JSON to HTML..." + python3 ${SCRIPT_DIR}/timestream-to-html.py \ + --vehicle-name ${VEHICLE} \ + --files ${TIMESTREAM_JSON_FILE} \ + --html-filename ${OUTPUT_FILE_HTML} \ + --include-signals "${INCLUDE_SIGNALS}" \ + --exclude-signals "${EXCLUDE_SIGNALS}" + else + echo "WARNING: Could not save timestream results for ${VEHICLE}, was any data collected?" + fi + done - for VEHICLE in ${VEHICLES[@]}; do - echo "Converting from Firehose JSON to HTML..." - OUTPUT_FILE_HTML="${COLLECTED_DATA_DIR}${VEHICLE}-json.html" - OUTPUT_FILE_S3_LINKS="${COLLECTED_DATA_DIR}${VEHICLE}-s3-links-json.txt" - python3 firehose-to-html.py \ - --vehicle-name ${VEHICLE} \ - --files ${COLLECTED_DATA_DIR}part-*.json \ - --html-filename ${OUTPUT_FILE_HTML} \ - --s3-links-filename ${OUTPUT_FILE_S3_LINKS} \ - --include-signals "${INCLUDED_SIGNALS}" \ - --exclude-signals "${EXCLUDED_SIGNALS}" - - if [ -s ${OUTPUT_FILE_S3_LINKS} ]; then - echo "Downloading the first 10 linked files..." - i=0 - cat ${OUTPUT_FILE_S3_LINKS} | while read LINE; do - if ((i < 10)); then - IMAGE_FILE=`basename ${LINE}.jpg` - # Remove random prefix so that filenames begin with date - IMAGE_FILE=`echo ${IMAGE_FILE} | sed -E 's/^[0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{12}\-//'` - aws s3 cp ${LINE} ${COLLECTED_DATA_DIR}${IMAGE_FILE} - fi - i=$((i+1)) + if [ ${#OUTPUT_FILES[@]} -eq 0 ]; then + echo "WARNING: No output files saved, was any data collected on any vehicles?" + else + echo "You can now view the collected data." + echo "----------------------------------------------------------------" + echo "| Collected data for all campaigns for ${NAME} in HTML format: |" + echo "----------------------------------------------------------------" + for FILE in ${OUTPUT_FILES[@]}; do + echo `realpath ${FILE}` done fi - echo "Converting from Firehose Parquet to HTML..." - OUTPUT_FILE_HTML="${COLLECTED_DATA_DIR}${VEHICLE}-parquet.html" - OUTPUT_FILE_S3_LINKS="${COLLECTED_DATA_DIR}${VEHICLE}-s3-links-parquet.txt" - python3 firehose-to-html.py \ - --vehicle-name ${VEHICLE} \ - --files ${COLLECTED_DATA_DIR}part-*.parquet \ - --html-filename ${OUTPUT_FILE_HTML} \ - --s3-links-filename ${OUTPUT_FILE_S3_LINKS} \ - --include-signals "${INCLUDED_SIGNALS}" \ - --exclude-signals "${EXCLUDED_SIGNALS}" - done - - echo "Zipping up collected data..." - zip ${COLLECTED_DATA_DIR}${NAME}.zip ${COLLECTED_DATA_DIR}* - - echo "You can now view the collected data." - echo "----------------------" - echo "| Collected S3 data: |" - echo "----------------------" - echo `realpath ${COLLECTED_DATA_DIR}${NAME}.zip` + else + echo "Error: Unknown data destination ${DATA_DESTINATION}" + exit -1 + fi fi diff --git a/tools/cloud/firehose-to-html.py b/tools/cloud/firehose-to-html.py index e925535c..8b840eb6 100755 --- a/tools/cloud/firehose-to-html.py +++ b/tools/cloud/firehose-to-html.py @@ -55,10 +55,22 @@ def process_row(vehicle_name, row, data, s3_links, include_list, exclude_list): raise Exception("Unsupported format") if row["vehicleName"] != vehicle_name: return - if "measure_value_DOUBLE" in row and not math.isnan(row["measure_value_DOUBLE"]): + if "measure_value_BOOLEAN" in row: + if not is_included(row["measure_name"], include_list, exclude_list): + return + expanded_data[row["measure_name"]] = 1 if row["measure_value_BOOLEAN"] else 0 + elif "measure_value_DOUBLE" in row and not math.isnan(row["measure_value_DOUBLE"]): if not is_included(row["measure_name"], include_list, exclude_list): return expanded_data[row["measure_name"]] = row["measure_value_DOUBLE"] + elif "measure_value_BIGINT" in row: + if not is_included(row["measure_name"], include_list, exclude_list): + return + expanded_data[row["measure_name"]] = row["measure_value_BIGINT"] + elif "measure_value_VARCHAR" in row: + if not is_included(row["measure_name"], include_list, exclude_list): + return + expanded_data[row["measure_name"]] = row["measure_value_VARCHAR"] elif "measure_value_STRUCT" in row and row["measure_value_STRUCT"] is not None: for struct_data in row["measure_value_STRUCT"].values(): if struct_data is None: @@ -73,7 +85,7 @@ def process_row(vehicle_name, row, data, s3_links, include_list, exclude_list): ) break else: - raise Exception("Unsupported format") + raise Exception(f"Unsupported format: {row}") data.append(expanded_data) diff --git a/tools/cloud/hscan.dbc b/tools/cloud/hscan.dbc index 9d2cf40e..eb675efb 100644 --- a/tools/cloud/hscan.dbc +++ b/tools/cloud/hscan.dbc @@ -33,17 +33,25 @@ BS_: BU_: +BO_ 769 Connectivity: 1 Vector__XXX + SG_ NetworkType : 7|8@0+ (1,0) [0|3] "" Vector__XXX + BO_ 401 ECM: 8 Vector__XXX SG_ DemoEngineTorque : 3|12@0+ (0.5,-848) [-848|1199.5] "Nm" Vector__XXX BO_ 532 ABS: 6 Vector__XXX SG_ DemoBrakePedalPressure : 0|8@0+ (75,0) [0|19125] "kPa" Vector__XXX +BO_ 1217 BCM: 8 Vector__XXX + SG_ DemoEngineCoolantTemperature : 23|8@0+ (1,-40) [-40|215] "deg C" Vector__XXX + BA_DEF_ SG_ "SignalType" STRING ; BA_DEF_ SG_ "SignalLongName" STRING ; BA_DEF_ BO_ "GenMsgCycleTime" INT 0 10000; BA_DEF_DEF_ "SignalType" ""; BA_DEF_DEF_ "SignalLongName" ""; BA_DEF_DEF_ "GenMsgCycleTime" 0; +BA_ "GenMsgCycleTime" BO_ 769 1000; BA_ "GenMsgCycleTime" BO_ 401 12; BA_ "GenMsgCycleTime" BO_ 532 50; +BA_ "GenMsgCycleTime" BO_ 1217 500; diff --git a/tools/cloud/install-deps.sh b/tools/cloud/install-deps.sh index 64419f74..fc548fb8 100755 --- a/tools/cloud/install-deps.sh +++ b/tools/cloud/install-deps.sh @@ -6,13 +6,16 @@ set -eo pipefail # On Ubuntu install Python 3 and pip if command -v apt &> /dev/null; then - apt update - apt install -y python3 python3-pip + apt update -o DPkg::Lock::Timeout=1800 + apt install -y -o DPkg::Lock::Timeout=1800 python3 python3-pip fi # Install pip packages # Note: CloudShell currently only supports pyarrow 12.0.1 -pip3 install \ + +python3 -m pip install importlib_metadata==8.5.0 + +python3 -m pip install \ wrapt==1.10.0 \ plotly==5.3.1 \ pandas==1.3.5 \ diff --git a/tools/cloud/network-interface-can.json b/tools/cloud/network-interface-can.json new file mode 100644 index 00000000..51957766 --- /dev/null +++ b/tools/cloud/network-interface-can.json @@ -0,0 +1,11 @@ +[ + { + "interfaceId": "1", + "type": "CAN_INTERFACE", + "canInterface": { + "name": "can0", + "protocolName": "CAN", + "protocolVersion": "2.0B" + } + } +] diff --git a/tools/cloud/network-interfaces.json b/tools/cloud/network-interface-obd.json similarity index 59% rename from tools/cloud/network-interfaces.json rename to tools/cloud/network-interface-obd.json index 1f369f76..3f684781 100644 --- a/tools/cloud/network-interfaces.json +++ b/tools/cloud/network-interface-obd.json @@ -9,14 +9,5 @@ "pidRequestIntervalSeconds": 5, "dtcRequestIntervalSeconds": 5 } - }, - { - "interfaceId": "1", - "type": "CAN_INTERFACE", - "canInterface": { - "name": "can0", - "protocolName": "CAN", - "protocolVersion": "2.0B" - } } ] diff --git a/tools/cloud/timestream-to-html.py b/tools/cloud/timestream-to-html.py index 99596927..a4eff627 100755 --- a/tools/cloud/timestream-to-html.py +++ b/tools/cloud/timestream-to-html.py @@ -92,8 +92,10 @@ def get_val(row, column): if val is None: val = get_val(row, "measure_value::bigint") if val is None: - val = get_val(row, "measure_value::boolean") != "false" - df.at[ts, signal_name] = float(val) + val = get_val(row, "measure_value::boolean") + if val is not None: + val = 0 if val == "false" else 1 + df.at[ts, signal_name] = val except Exception as e: raise Exception(str(e) + f" in {file.name}") diff --git a/tools/configure-fwe.sh b/tools/configure-fwe.sh index a41b4edf..7c8aaa1b 100755 --- a/tools/configure-fwe.sh +++ b/tools/configure-fwe.sh @@ -4,57 +4,27 @@ set -euo pipefail -if [ -z "${INPUT_CONFIG_FILE+x}" ]; then - INPUT_CONFIG_FILE="" -fi -if [ -z "${OUTPUT_CONFIG_FILE+x}" ]; then - OUTPUT_CONFIG_FILE="" -fi -if [ -z "${CERTIFICATE+x}" ]; then - CERTIFICATE="" -fi -if [ -z "${CERTIFICATE_FILE+x}" ]; then - CERTIFICATE_FILE="/etc/aws-iot-fleetwise/certificate.pem" -fi -if [ -z "${PRIVATE_KEY+x}" ]; then - PRIVATE_KEY="" -fi -if [ -z "${PRIVATE_KEY_FILE+x}" ]; then - PRIVATE_KEY_FILE="/etc/aws-iot-fleetwise/private-key.key" -fi -if [ -z "${VEHICLE_NAME+x}" ]; then - VEHICLE_NAME="" -fi -if [ -z "${ENDPOINT_URL+x}" ]; then - ENDPOINT_URL="" -fi -if [ -z "${CAN_BUS0+x}" ]; then - CAN_BUS0="" -fi -if [ -z "${LOG_LEVEL+x}" ]; then - LOG_LEVEL="Info" -fi -if [ -z "${LOG_COLOR+x}" ]; then - LOG_COLOR="Auto" -fi -if [ -z "${PERSISTENCY_PATH+x}" ]; then - PERSISTENCY_PATH="/var/aws-iot-fleetwise/" -fi -if [ -z "${TOPIC_PREFIX+x}" ]; then - TOPIC_PREFIX="\$aws/iotfleetwise/" -fi -if [ -z "${CONNECTION_TYPE+x}" ]; then - CONNECTION_TYPE="iotCore" -fi -if [ -z "${CREDS_ENDPOINT_URL+x}" ]; then - CREDS_ENDPOINT_URL="" -fi -if [ -z "${CREDS_ROLE_ALIAS+x}" ]; then - CREDS_ROLE_ALIAS="" -fi -if [ -z "${RAW_DATA_BUFFER_SIZE+x}" ]; then - RAW_DATA_BUFFER_SIZE=1073741824 -fi +# Allow options to also be set as env vars, but set default values if missing +: ${INPUT_CONFIG_FILE:=""} +: ${OUTPUT_CONFIG_FILE:=""} +: ${CERTIFICATE:=""} +: ${CERTIFICATE_FILE:="/etc/aws-iot-fleetwise/certificate.pem"} +: ${PRIVATE_KEY:=""} +: ${PRIVATE_KEY_FILE:="/etc/aws-iot-fleetwise/private-key.key"} +: ${VEHICLE_NAME:=""} +: ${ENDPOINT_URL:=""} +: ${CAN_BUS0:=""} +: ${LOG_LEVEL:="Info"} +: ${LOG_COLOR:="Auto"} +: ${PERSISTENCY_PATH:="/var/aws-iot-fleetwise/"} +: ${TOPIC_PREFIX:="\$aws/iotfleetwise/"} +: ${CONNECTION_TYPE:="iotCore"} +: ${KEEP_ALIVE_INTERVAL_SECONDS:="60"} +: ${PING_TIMEOUT_MS:="30000"} +: ${SESSION_EXPIRY_INTERVAL_SECONDS:="0"} +: ${CREDS_ENDPOINT_URL:=""} +: ${CREDS_ROLE_ALIAS:=""} +: ${RAW_DATA_BUFFER_SIZE:=1073741824} parse_args() { while [ "$#" -gt 0 ]; do @@ -71,6 +41,18 @@ parse_args() { CONNECTION_TYPE=$2 shift ;; + --keep-alive-interval-seconds) + KEEP_ALIVE_INTERVAL_SECONDS=$2 + shift + ;; + --ping-timeout-ms) + PING_TIMEOUT_MS=$2 + shift + ;; + --session-expiry-interval-seconds) + SESSION_EXPIRY_INTERVAL_SECONDS=$2 + shift + ;; --vehicle-name) VEHICLE_NAME=$2 shift @@ -129,23 +111,26 @@ parse_args() { ;; --help) echo "Usage: $0 [OPTION]" - echo " --input-config-file Input JSON config file" - echo " --output-config-file Output JSON config file" - echo " --connection-type Connectivity connection type, default: ${CONNECTION_TYPE}" - echo " --vehicle-name Vehicle name" - echo " --endpoint-url IoT Core MQTT endpoint URL" - echo " --can-bus0 CAN bus 0, e.g. vcan0" - echo " --certificate Certificate" - echo " --certificate-file Certificate file, default: ${CERTIFICATE_FILE}" - echo " --private-key Private key" - echo " --private-key-file Private key file, default: ${PRIVATE_KEY_FILE}" - echo " --persistency-path Persistency path, default: ${PERSISTENCY_PATH}" - echo " --topic-prefix IoT MQTT topic prefix, default: ${TOPIC_PREFIX}" - echo " --log-level Log level. Either: Off, Error, Warning, Info, Trace. Default: ${LOG_LEVEL}" - echo " --log-color Whether logs should be colored. Either: Auto, Yes, No. Default: ${LOG_COLOR}" - echo " --creds-endpoint-url Endpoint URL for AWS IoT Credentials Provider" - echo " --creds-role-alias Role alias for AWS IoT Credentials Provider" - echo " --raw-data-buffer-size Raw data buffer size, default: ${RAW_DATA_BUFFER_SIZE}" + echo " --input-config-file Input JSON config file" + echo " --output-config-file Output JSON config file" + echo " --connection-type Connectivity connection type, default: ${CONNECTION_TYPE}" + echo " --keep-alive-interval-seconds (\"iotCore\" connection type only) Keep alive interval for MQTT client, default: ${KEEP_ALIVE_INTERVAL_SECONDS}" + echo " --ping-timeout-ms (\"iotCore\" connection type only) Ping timeout for MQTT client, default: ${PING_TIMEOUT_MS}" + echo " --session-expiry-interval-seconds (\"iotCore\" connection type only) Expiry interval for persistent sessions, 0 means disabled, default: ${SESSION_EXPIRY_INTERVAL_SECONDS}" + echo " --vehicle-name Vehicle name" + echo " --endpoint-url IoT Core MQTT endpoint URL" + echo " --can-bus0 CAN bus 0, e.g. vcan0" + echo " --certificate Certificate" + echo " --certificate-file Certificate file, default: ${CERTIFICATE_FILE}" + echo " --private-key Private key" + echo " --private-key-file Private key file, default: ${PRIVATE_KEY_FILE}" + echo " --persistency-path Persistency path, default: ${PERSISTENCY_PATH}" + echo " --topic-prefix IoT MQTT topic prefix, default: ${TOPIC_PREFIX}" + echo " --log-level Log level. Either: Off, Error, Warning, Info, Trace. Default: ${LOG_LEVEL}" + echo " --log-color Whether logs should be colored. Either: Auto, Yes, No. Default: ${LOG_COLOR}" + echo " --creds-endpoint-url Endpoint URL for AWS IoT Credentials Provider" + echo " --creds-role-alias Role alias for AWS IoT Credentials Provider" + echo " --raw-data-buffer-size Raw data buffer size, default: ${RAW_DATA_BUFFER_SIZE}" exit 0 ;; esac @@ -203,12 +188,18 @@ if [ "$CONNECTION_TYPE" == "iotCore" ]; then | if [[ $PRIVATE_KEY ]]; \ then jq "del(.staticConfig.mqttConnection.privateKeyFilename)" | jq ".staticConfig.mqttConnection.privateKey=\"${PRIVATE_KEY}\""; \ else jq ".staticConfig.mqttConnection.privateKeyFilename=\"${PRIVATE_KEY_FILE}\""; \ - fi` + fi \ + | jq ".staticConfig.mqttConnection.keepAliveIntervalSeconds=${KEEP_ALIVE_INTERVAL_SECONDS}" \ + | jq ".staticConfig.mqttConnection.pingTimeoutMs=${PING_TIMEOUT_MS}" \ + | jq ".staticConfig.mqttConnection.sessionExpiryIntervalSeconds=${SESSION_EXPIRY_INTERVAL_SECONDS}"` elif [ "$CONNECTION_TYPE" == "iotGreengrassV2" ]; then OUTPUT_CONFIG=`echo "${OUTPUT_CONFIG}" \ | jq "del(.staticConfig.mqttConnection.endpointUrl)" \ | jq "del(.staticConfig.mqttConnection.certificateFilename)" \ - | jq "del(.staticConfig.mqttConnection.privateKeyFilename)"` + | jq "del(.staticConfig.mqttConnection.privateKeyFilename)" \ + | jq "del(.staticConfig.mqttConnection.keepAliveIntervalSeconds)" \ + | jq "del(.staticConfig.mqttConnection.pingTimeoutMs)" \ + | jq "del(.staticConfig.mqttConnection.sessionExpiryIntervalSeconds)"` else echo "Error: Unknown connection type ${CONNECTION_TYPE}" fi diff --git a/tools/install-cansim.sh b/tools/install-cansim.sh index 8ab050f9..49c539a1 100755 --- a/tools/install-cansim.sh +++ b/tools/install-cansim.sh @@ -26,8 +26,8 @@ parse_args() { parse_args "$@" # Install Python 3 and pip -apt update -apt install -y python3 python3-pip +apt update -o DPkg::Lock::Timeout=120 +apt install -y -o DPkg::Lock::Timeout=120 python3 python3-pip # Install pip packages pip3 install \ diff --git a/tools/install-deps-cross-android.sh b/tools/install-deps-cross-android.sh index df6aee12..d39d7f66 100755 --- a/tools/install-deps-cross-android.sh +++ b/tools/install-deps-cross-android.sh @@ -43,8 +43,8 @@ parse_args() { parse_args "$@" -apt update -apt install -y \ +apt update -o DPkg::Lock::Timeout=1800 +apt install -y -o DPkg::Lock::Timeout=1800 \ unzip \ git \ wget \ @@ -70,9 +70,7 @@ if [ ! -d ${SDK_PREFIX} ]; then fi export PATH=${SDK_PREFIX}/cmake/${VERSION_CMAKE}/bin/:${PATH} -if [ -z "${NATIVE_PREFIX+x}" ]; then - NATIVE_PREFIX="/usr/local/`gcc -dumpmachine`" -fi +: ${NATIVE_PREFIX:="/usr/local/`gcc -dumpmachine`"} install_deps() { TARGET_ARCH="$1" @@ -90,7 +88,7 @@ install_deps() { wget -q https://archives.boost.io/release/${VERSION_BOOST}/source/boost_${VERSION_BOOST//./_}.tar.bz2 -O Boost-for-Android/boost_${VERSION_BOOST//./_}.tar.bz2 fi cd Boost-for-Android - git checkout 53e6c16ea80c7dcb2683fd548e0c7a09ddffbfc1 + git checkout ${VERSION_BOOST_FOR_ANDROID} rm -rf build ./build-android.sh \ ${SDK_PREFIX}/ndk/${VERSION_ANDROID_NDK} \ @@ -271,7 +269,6 @@ install_deps() { -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_ONLY='transfer;s3-crt;iot' \ - -DAWS_CUSTOM_MEMORY_MANAGEMENT=ON \ -DANDROID_ABI=${TARGET_ARCH} \ -DANDROID_PLATFORM=android-${VERSION_ANDROID_API} \ -DCMAKE_ANDROID_NDK=${SDK_PREFIX}/ndk/${VERSION_ANDROID_NDK} \ diff --git a/tools/install-deps-cross-arm64.sh b/tools/install-deps-cross-arm64.sh index c54454a3..de6311f1 100755 --- a/tools/install-deps-cross-arm64.sh +++ b/tools/install-deps-cross-arm64.sh @@ -76,10 +76,10 @@ sed -i -E "s#(archive|security).ubuntu.com/ubuntu#ports.ubuntu.com/ubuntu-ports# print_file "After patching" /etc/apt/sources.list.d/arm64.list dpkg --add-architecture arm64 -apt update -apt install -y \ +apt update -o DPkg::Lock::Timeout=120 +apt install -y -o DPkg::Lock::Timeout=120 \ build-essential -apt install -y \ +apt install -y -o DPkg::Lock::Timeout=120 \ cmake \ crossbuild-essential-arm64 \ curl \ @@ -93,8 +93,8 @@ apt install -y \ if ${WITH_ROS2_SUPPORT}; then wget -q https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -O /usr/share/keyrings/ros-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(source /etc/os-release && echo $UBUNTU_CODENAME) main" > /etc/apt/sources.list.d/ros2.list - apt update - apt install -y \ + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 \ bison \ python3-numpy \ python3-lark \ @@ -102,7 +102,8 @@ if ${WITH_ROS2_SUPPORT}; then libacl1-dev:arm64 \ liblog4cxx-dev:arm64 \ libpython3-dev:arm64 \ - python3-colcon-common-extensions + python3-colcon-common-extensions \ + ros-dev-tools fi if [ ! -f /usr/include/linux/can/isotp.h ]; then @@ -114,9 +115,7 @@ if [ ! -f /usr/include/linux/can/isotp.h ]; then rm -rf can-isotp fi -if [ -z "${NATIVE_PREFIX+x}" ]; then - NATIVE_PREFIX="/usr/local/`gcc -dumpmachine`" -fi +: ${NATIVE_PREFIX:="/usr/local/`gcc -dumpmachine`"} if ! ${USE_CACHE} || [ ! -d /usr/local/aarch64-linux-gnu ] || [ ! -d ${NATIVE_PREFIX} ]; then mkdir -p /usr/local/aarch64-linux-gnu/lib/cmake/ @@ -274,7 +273,6 @@ if ! ${USE_CACHE} || [ ! -d /usr/local/aarch64-linux-gnu ] || [ ! -d ${NATIVE_PR -DBUILD_SHARED_LIBS=${SHARED_LIBS} \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_ONLY='transfer;s3-crt;iot' \ - -DAWS_CUSTOM_MEMORY_MANAGEMENT=ON \ ${AWS_SDK_CPP_OPTIONS} \ -DCMAKE_TOOLCHAIN_FILE=/usr/local/aarch64-linux-gnu/lib/cmake/arm64-toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=/usr/local/aarch64-linux-gnu \ @@ -311,8 +309,6 @@ if ! ${USE_CACHE} || [ ! -d /usr/local/aarch64-linux-gnu ] || [ ! -d ${NATIVE_PR make install -j`nproc` cd ../.. - apt install -y \ - ros-dev-tools git clone -b 0.8.0 https://github.com/eclipse-cyclonedds/cyclonedds.git cd cyclonedds mkdir build && cd build diff --git a/tools/install-deps-cross-armhf.sh b/tools/install-deps-cross-armhf.sh index ae346687..e273f049 100755 --- a/tools/install-deps-cross-armhf.sh +++ b/tools/install-deps-cross-armhf.sh @@ -76,10 +76,10 @@ sed -i -E "s#(archive|security).ubuntu.com/ubuntu#ports.ubuntu.com/ubuntu-ports# print_file "After patching" /etc/apt/sources.list.d/armhf.list dpkg --add-architecture armhf -apt update -apt install -y \ +apt update -o DPkg::Lock::Timeout=120 +apt install -y -o DPkg::Lock::Timeout=120 \ build-essential -apt install -y \ +apt install -y -o DPkg::Lock::Timeout=120 \ cmake \ crossbuild-essential-armhf \ curl \ @@ -93,8 +93,8 @@ apt install -y \ if ${WITH_ROS2_SUPPORT}; then wget -q https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -O /usr/share/keyrings/ros-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(source /etc/os-release && echo $UBUNTU_CODENAME) main" > /etc/apt/sources.list.d/ros2.list - apt update - apt install -y \ + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 \ bison \ python3-numpy \ python3-lark \ @@ -102,7 +102,8 @@ if ${WITH_ROS2_SUPPORT}; then libacl1-dev:armhf \ liblog4cxx-dev:armhf \ libpython3-dev:armhf \ - python3-colcon-common-extensions + python3-colcon-common-extensions \ + ros-dev-tools fi if [ ! -f /usr/include/linux/can/isotp.h ]; then @@ -114,9 +115,7 @@ if [ ! -f /usr/include/linux/can/isotp.h ]; then rm -rf can-isotp fi -if [ -z "${NATIVE_PREFIX+x}" ]; then - NATIVE_PREFIX="/usr/local/`gcc -dumpmachine`" -fi +: ${NATIVE_PREFIX:="/usr/local/`gcc -dumpmachine`"} if ! ${USE_CACHE} || [ ! -d /usr/local/arm-linux-gnueabihf ] || [ ! -d ${NATIVE_PREFIX} ]; then mkdir -p /usr/local/arm-linux-gnueabihf/lib/cmake/ @@ -274,7 +273,6 @@ if ! ${USE_CACHE} || [ ! -d /usr/local/arm-linux-gnueabihf ] || [ ! -d ${NATIVE_ -DBUILD_SHARED_LIBS=${SHARED_LIBS} \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_ONLY='transfer;s3-crt;iot' \ - -DAWS_CUSTOM_MEMORY_MANAGEMENT=ON \ ${AWS_SDK_CPP_OPTIONS} \ -DCMAKE_TOOLCHAIN_FILE=/usr/local/arm-linux-gnueabihf/lib/cmake/armhf-toolchain.cmake \ -DCMAKE_INSTALL_PREFIX=/usr/local/arm-linux-gnueabihf \ @@ -311,8 +309,6 @@ if ! ${USE_CACHE} || [ ! -d /usr/local/arm-linux-gnueabihf ] || [ ! -d ${NATIVE_ make install -j`nproc` cd ../.. - apt install -y \ - ros-dev-tools git clone -b 0.8.0 https://github.com/eclipse-cyclonedds/cyclonedds.git cd cyclonedds mkdir build && cd build diff --git a/tools/install-deps-native.sh b/tools/install-deps-native.sh index 3e54f1eb..31374d36 100755 --- a/tools/install-deps-native.sh +++ b/tools/install-deps-native.sh @@ -52,8 +52,8 @@ parse_args() { parse_args "$@" if ${INSTALL_BUILD_TIME_DEPS}; then - apt update - apt install -y \ + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 \ build-essential \ clang-tidy-12 \ cmake \ @@ -70,17 +70,17 @@ if ${INSTALL_BUILD_TIME_DEPS}; then fi if ${WITH_ROS2_SUPPORT}; then - command -v wget > /dev/null || ( apt update && apt install -y wget ) + command -v wget > /dev/null || ( apt update -o DPkg::Lock::Timeout=120 && apt install -y -o DPkg::Lock::Timeout=120 wget) wget -q https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -O /usr/share/keyrings/ros-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(source /etc/os-release && echo $UBUNTU_CODENAME) main" > /etc/apt/sources.list.d/ros2.list - apt update - apt install -y \ + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 \ ros-galactic-rclcpp \ ros-galactic-rosbag2 \ ros-galactic-ros2topic \ ros-galactic-sensor-msgs if ${INSTALL_BUILD_TIME_DEPS}; then - apt install -y \ + apt install -y -o DPkg::Lock::Timeout=120 \ ros-galactic-rosidl-default-generators \ python3-colcon-common-extensions fi @@ -95,9 +95,7 @@ if ${INSTALL_BUILD_TIME_DEPS} && [ ! -f /usr/include/linux/can/isotp.h ]; then rm -rf can-isotp fi -if [ -z "${PREFIX+x}" ]; then - PREFIX="/usr/local/`gcc -dumpmachine`" -fi +: ${PREFIX:="/usr/local/`gcc -dumpmachine`"} if ${INSTALL_BUILD_TIME_DEPS} && ( ! ${USE_CACHE} || [ ! -d ${PREFIX} ] ); then mkdir -p ${PREFIX} @@ -247,7 +245,6 @@ if ${INSTALL_BUILD_TIME_DEPS} && ( ! ${USE_CACHE} || [ ! -d ${PREFIX} ] ); then -DBUILD_SHARED_LIBS=${SHARED_LIBS} \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_ONLY='transfer;s3-crt;iot' \ - -DAWS_CUSTOM_MEMORY_MANAGEMENT=ON \ ${AWS_SDK_CPP_OPTIONS} \ -DCMAKE_INSTALL_PREFIX=${PREFIX} \ .. diff --git a/tools/install-deps-versions.sh b/tools/install-deps-versions.sh index 5e22021f..7592f240 100755 --- a/tools/install-deps-versions.sh +++ b/tools/install-deps-versions.sh @@ -1,13 +1,18 @@ #!/bin/bash -export VERSION_BOOST="1.78.0" +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +export VERSION_BOOST="1.84.0" +# This should be a commit from https://github.com/moritz-wundke/Boost-for-Android that provides +# support for the desired boost version. +export VERSION_BOOST_FOR_ANDROID="83ba7174dea6e0595085dc30051eb17747e7260a" export VERSION_JSON_CPP="1.9.5" export VERSION_PROTOBUF="3.21.12" export VERSION_PROTOBUF_RELEASE="v21.12" export VERSION_CURL="7.86.0" export VERSION_CURL_RELEASE="curl-7_86_0" -export VERSION_AWS_SDK_CPP="1.11.177" +export VERSION_AWS_SDK_CPP="1.11.284" # MAKE SURE THE CRT VERSION IN AWS_SDK_CPP IS THE SAME AS IN AWS_IOT_SDK_CPP -export VERSION_AWS_IOT_DEVICE_SDK_CPP_V2="v1.30.0" +export VERSION_AWS_IOT_DEVICE_SDK_CPP_V2="v1.32.2" export VERSION_TINYXML2="6.0.0" export VERSION_FAST_CDR="v1.0.21" export VERSION_GOOGLE_TEST="release-1.10.0" diff --git a/tools/install-deps-yocto.sh b/tools/install-deps-yocto.sh index 9552d29b..9bcf0eea 100755 --- a/tools/install-deps-yocto.sh +++ b/tools/install-deps-yocto.sh @@ -6,8 +6,8 @@ set -euo pipefail export DEBIAN_FRONTEND="noninteractive" -apt update -apt install -y \ +apt update -o DPkg::Lock::Timeout=120 +apt install -y -o DPkg::Lock::Timeout=120 \ gawk wget git-core diffstat unzip texinfo \ build-essential chrpath socat cpio python3 python3-pip python3-pexpect \ xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev \ @@ -15,7 +15,7 @@ apt install -y \ zstd lz4 if [ `dpkg --print-architecture` == "amd64" ]; then - apt install -y gcc-multilib + apt install -y -o DPkg::Lock::Timeout=120 gcc-multilib fi # Google 'repo' tool, see https://source.android.com/setup/develop/repo diff --git a/tools/install-socketcan.sh b/tools/install-socketcan.sh index 69f3ac32..9f279c5f 100755 --- a/tools/install-socketcan.sh +++ b/tools/install-socketcan.sh @@ -27,14 +27,14 @@ parse_args "$@" if ! command -v cansend > /dev/null; then # Install packages - apt update - apt install -y can-utils + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 can-utils fi # For EC2, the SocketCAN modules vcan and can-gw are included in a separate package: if uname -r | grep -q aws; then - apt update - apt install -y linux-modules-extra-aws + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 linux-modules-extra-aws fi # Install can-isotp kernel module if not installed @@ -44,8 +44,8 @@ echo "Installing kernel module: $MODULE" if modinfo $MODULE &> /dev/null ; then echo "$MODULE is already in system. There is no need to install it." else - apt update - apt install -y build-essential dkms git + apt update -o DPkg::Lock::Timeout=120 + apt install -y -o DPkg::Lock::Timeout=120 build-essential dkms git git clone https://github.com/hartkopp/can-isotp.git cd can-isotp git checkout beb4650660179963a8ed5b5cbf2085cc1b34f608 diff --git a/tools/provision.sh b/tools/provision.sh index db7a5658..e901e912 100755 --- a/tools/provision.sh +++ b/tools/provision.sh @@ -111,33 +111,35 @@ fi NAME="${VEHICLE_NAME}-${TIMESTAMP}" -if [ ${ONLY_CLEAN_UP} == true ]; then - - PRINCIPAL_ARN=$(aws iot list-thing-principals --thing-name ${VEHICLE_NAME} --region ${REGION} | jq -r ".principals[0]") +if ${ONLY_CLEAN_UP}; then + PRINCIPAL_ARN=$(aws iot list-thing-principals --thing-name ${VEHICLE_NAME} --region ${REGION} ${ENDPOINT_URL_OPTION} | jq -r ".principals[0]") CERTIFICATE_ID=$(echo ${PRINCIPAL_ARN} | cut -d'/' -f2 ) - POLICY_NAME=$(aws iot list-principal-policies --principal ${PRINCIPAL_ARN} --region ${REGION} | jq -r ".policies[0].policyName") + POLICY_NAME=$(aws iot list-principal-policies --principal ${PRINCIPAL_ARN} --region ${REGION} ${ENDPOINT_URL_OPTION} | jq -r ".policies[0].policyName") RETRY_COUNTER=10 - while ! aws iot delete-thing --thing-name ${VEHICLE_NAME} --region ${REGION} - do + while true; do # Delete depending resources - aws iot detach-thing-principal --thing-name ${VEHICLE_NAME} --principal ${PRINCIPAL_ARN} --region ${REGION} || true - aws iot detach-principal-policy --policy-name "${POLICY_NAME}" --principal ${PRINCIPAL_ARN} --region ${REGION} || true - aws iot delete-policy --policy-name "${POLICY_NAME}" --region ${REGION} || true - aws iot update-certificate --certificate-id ${CERTIFICATE_ID} --new-status INACTIVE --region ${REGION} || true - aws iot delete-certificate --certificate-id ${CERTIFICATE_ID} --region ${REGION} || true - - echo "Wait one second before next retry" - sleep 1 - if [ "${RETRY_COUNTER}" -eq "0" ]; then - echo "Failed aws iot delete-thing" - exit 1 + aws iot detach-thing-principal --thing-name ${VEHICLE_NAME} --principal ${PRINCIPAL_ARN} --region ${REGION} ${ENDPOINT_URL_OPTION} || true + aws iot detach-principal-policy --policy-name "${POLICY_NAME}" --principal ${PRINCIPAL_ARN} --region ${REGION} ${ENDPOINT_URL_OPTION} || true + aws iot delete-policy --policy-name "${POLICY_NAME}" --region ${REGION} ${ENDPOINT_URL_OPTION} || true + aws iot update-certificate --certificate-id ${CERTIFICATE_ID} --new-status INACTIVE --region ${REGION} ${ENDPOINT_URL_OPTION} || true + aws iot delete-certificate --certificate-id ${CERTIFICATE_ID} --region ${REGION} ${ENDPOINT_URL_OPTION} || true + if DELETE_THING_STATUS=`aws iot delete-thing --thing-name ${VEHICLE_NAME} --region ${REGION} ${ENDPOINT_URL_OPTION}`; then + break + elif ! echo ${DELETE_THING_STATUS} | grep -q "InvalidRequestException"; then + echo "${DELETE_THING_STATUS}" >&2 + exit -1 + else + if [ "${RETRY_COUNTER}" -eq "0" ]; then + echo "Failed aws iot delete-thing" + exit 1 + fi + ((RETRY_COUNTER--)) + sleep 1 fi - ((RETRY_COUNTER--)) done echo "Successfully deleted iot thing" exit 0 - fi echo -n "Date: " diff --git a/tools/renesas-rcar-s4/make-rootfs.sh b/tools/renesas-rcar-s4/make-rootfs.sh index f3887155..3201999b 100755 --- a/tools/renesas-rcar-s4/make-rootfs.sh +++ b/tools/renesas-rcar-s4/make-rootfs.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. set -euo pipefail @@ -45,7 +46,7 @@ if [[ $# < 2 ]]; then echo " SD card option(optional) For S4 Spider:" echo " -sd: use sdcard instead of eMMC." echo "Required package:" - echo " sudo apt install debootstrap qemu-user-static binfmt-support gcc-aarch64-linux-gnu" + echo " sudo apt install -o DPkg::Lock::Timeout=120 debootstrap qemu-user-static binfmt-support gcc-aarch64-linux-gnu" echo " # Also required the software which is needed to build linux kernel." echo "memo:" echo " debootstrap is not needed, but dependencies are required." @@ -72,7 +73,7 @@ fi if [[ "`update-binfmts --display | grep aarch64`" == "" ]]; then echo "qemu may not be installed." echo "Please install it by following command:" - echo " sudo apt install qemu-user-static" + echo " sudo apt install -o DPkg::Lock::Timeout=120 qemu-user-static" exit fi if [[ "`update-binfmts --display | grep aarch64 | grep enable`" == "" ]]; then @@ -177,12 +178,12 @@ cp ${QEMU_BIN_PATH} ./${ROOTFS}/${QEMU_BIN_PATH} chroot "${ROOTFS}" sh -c " \ export DEBIAN_FRONTEND=noninteractive \ && echo nameserver ${NAMESERVER} >/etc/resolv.conf \ - && apt update \ - && apt upgrade -y \ - && apt install -y apt-utils perl-modules \ - && apt install -y ubuntu-standard \ - && apt install -y vim net-tools ssh sudo tzdata rsyslog udev iputils-ping \ - && apt install -y unzip curl kmod iproute2 git python3-pip nano \ + && apt update -o DPkg::Lock::Timeout=120 \ + && apt upgrade -y -o DPkg::Lock::Timeout=120 \ + && apt install -y -o DPkg::Lock::Timeout=120 apt-utils perl-modules \ + && apt install -y -o DPkg::Lock::Timeout=120 ubuntu-standard \ + && apt install -y -o DPkg::Lock::Timeout=120 vim net-tools ssh sudo tzdata rsyslog udev iputils-ping \ + && apt install -y -o DPkg::Lock::Timeout=120 unzip curl kmod iproute2 git python3-pip nano \ && echo \"${DHCP_CONF}\" > /etc/systemd/network/01-${NET_DEV}.network \ && useradd -m -s /bin/bash -G sudo ${USERNAME} \ && echo ${USERNAME}:${USERNAME} | chpasswd \ diff --git a/tools/test-fwe.sh b/tools/test-fwe.sh index bc510fae..3dc96d51 100755 --- a/tools/test-fwe.sh +++ b/tools/test-fwe.sh @@ -2,26 +2,25 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -set -eo pipefail +set -efo pipefail WITH_ROS2_SUPPORT="false" -EXTRA_CTEST_ARGS="" +EXTRA_GTEST_FILTER="*" parse_args() { while [ "$#" -gt 0 ]; do case $1 in --with-ros2-support) WITH_ROS2_SUPPORT="true" - shift ;; - --extra-ctest-args) - EXTRA_CTEST_ARGS=$2 + --extra-gtest-filter) + EXTRA_GTEST_FILTER=$2 shift ;; --help) echo "Usage: $0 [OPTION]" - echo " --with-ros2-support Test with ROS2 support" - echo " --extra-ctest-args Args to be passed to ctest eg. -E " + echo " --with-ros2-support Test with ROS2 support" + echo " --extra-gtest-filter Regex to set for gtest filter eg. '-**' or '**'" exit 0 ;; esac @@ -37,4 +36,4 @@ if ${WITH_ROS2_SUPPORT}; then else cd build fi -ctest -V ${EXTRA_CTEST_ARGS} +GTEST_FILTER=${EXTRA_GTEST_FILTER} ctest -V diff --git a/tools/yocto/sources/meta-aws-iot-fleetwise/recipes-extended/setup-socketcan/files/setup-socketcan.sh b/tools/yocto/sources/meta-aws-iot-fleetwise/recipes-extended/setup-socketcan/files/setup-socketcan.sh index 25974e43..e146d14a 100755 --- a/tools/yocto/sources/meta-aws-iot-fleetwise/recipes-extended/setup-socketcan/files/setup-socketcan.sh +++ b/tools/yocto/sources/meta-aws-iot-fleetwise/recipes-extended/setup-socketcan/files/setup-socketcan.sh @@ -1,4 +1,6 @@ #!/bin/sh +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + ip link add dev vcan0 type vcan; ip link set up vcan0 ip link add dev vcan1 type vcan; ip link set up vcan1 ip link set up can0 txqueuelen 1000 type can bitrate 500000 dbitrate 5000000 fd on restart-ms 100