aws · olgaakulinushkina · Oct 30, 2024 · Oct 30, 2024 · Oct 30, 2024
@@ -62,7 +62,7 @@ runs:
       shell: bash
       if: inputs.build-arch  == 'native'
       run: |
-        ./tools/test-fwe.sh ${{ inputs.extra-options }} --extra-ctest-args "-E \"CANDataSourceTest|ISOTPOverCANProtocolTest|IoTFleetWiseEngineTest|OBDOverCANModuleTest\""
+        ./tools/test-fwe.sh ${{ inputs.extra-options }} --extra-gtest-filter "-*CANDataSourceTest*:*ISOTPOverCANProtocolTest*:*IoTFleetWiseEngineTest*:*OBDOverCANModuleTest*"
 
     - name: upload-artifacts
       uses: actions/upload-artifact@v4

@@ -37,6 +37,7 @@ jobs:
         with:
           build-arch: "native"
           upload-arch: "amd64"
+          extra-options: "--with-greengrassv2-support"
           dist-name: "aws-iot-fleetwise-edge"
           cache-paths: /usr/local/x86_64-linux-gnu
           dist-files: build/aws-iot-fleetwise-edge:.
@@ -49,6 +50,7 @@ jobs:
         with:
           build-arch: "cross-arm64"
           upload-arch: "arm64"
+          extra-options: "--with-greengrassv2-support"
           dist-name: "aws-iot-fleetwise-edge"
           cache-paths: /usr/local/aarch64-linux-gnu:/usr/local/x86_64-linux-gnu
           dist-files: build/aws-iot-fleetwise-edge:.
@@ -61,7 +63,7 @@ jobs:
         with:
           build-arch: "cross-armhf"
           upload-arch: "armhf"
-          extra-options: "--with-iwave-gps-support"
+          extra-options: "--with-greengrassv2-support --with-iwave-gps-support"
           dist-name: "aws-iot-fleetwise-edge"
           cache-paths: /usr/local/arm-linux-gnueabihf:/usr/local/x86_64-linux-gnu
           dist-files: build/aws-iot-fleetwise-edge:.
@@ -74,7 +76,7 @@ jobs:
         with:
           build-arch: "native"
           upload-arch: "amd64"
-          extra-options: "--with-ros2-support"
+          extra-options: "--with-greengrassv2-support --with-ros2-support"
           dist-name: "aws-iot-fleetwise-edge-ros2"
           cache-paths: /usr/local/x86_64-linux-gnu:/opt/ros
           dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:.
@@ -87,7 +89,7 @@ jobs:
         with:
           build-arch: "cross-arm64"
           upload-arch: "arm64"
-          extra-options: "--with-ros2-support"
+          extra-options: "--with-greengrassv2-support --with-ros2-support"
           dist-name: "aws-iot-fleetwise-edge-ros2"
           cache-paths: /usr/local/aarch64-linux-gnu:/usr/local/x86_64-linux-gnu:/opt/ros
           dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:.
@@ -100,7 +102,7 @@ jobs:
         with:
           build-arch: "cross-armhf"
           upload-arch: "armhf"
-          extra-options: "--with-ros2-support"
+          extra-options: "--with-greengrassv2-support --with-ros2-support"
           dist-name: "aws-iot-fleetwise-edge-ros2"
           cache-paths: /usr/local/arm-linux-gnueabihf:/usr/local/x86_64-linux-gnu:/opt/ros
           dist-files: build/iotfleetwise/aws-iot-fleetwise-edge:.

@@ -1,3 +1,5 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 default_language_version:
@@ -121,10 +123,17 @@ repos:
         verbose: true
       - id: validate-build-files
         name: validate-build-files
-        entry:
-          bash -e -c 'FILE_MISSING=false; for FILE in "$@"; do if ! grep -q "${FILE}"
-          CMakeLists.txt; then echo "${FILE} is missing from CMakeLists.txt"; FILE_MISSING=true; fi;
-          done; if ${FILE_MISSING}; then exit -1; fi' --
+        entry: |
+          bash -e -c '
+            VIOLATION=false
+            for FILE in "$@"; do
+              if ! grep -q "${FILE}" CMakeLists.txt; then
+                echo "${FILE} is missing from CMakeLists.txt"
+                VIOLATION=true
+              fi
+            done
+            if ${VIOLATION}; then exit -1; fi
+          ' --
         language: python
         files: "^src|^test/unit"
         exclude: "^test/unit/support"
@@ -141,3 +150,20 @@ repos:
         language: pygrep
         entry: "[\\x80-\\xFF]"
         types: [text]
+      - id: ensure-copyright-header
+        name: ensure-copyright-header
+        entry: |
+          bash -e -c '
+            VIOLATION=false
+            for FILE in "$@"; do
+              if ! head -n2 "${FILE}" | grep -q "Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved."; then
+                echo "${FILE} is missing copyright headers"
+                VIOLATION=true
+              fi
+            done
+            if ${VIOLATION}; then exit -1; fi
+          ' --
+        language: python
+        types_or: [c++, python, yaml, cmake, proto, shell]
+        exclude: "gradlew|.prettierrc.yaml|.clang-format|.clang-tidy|create_avd_config.sh"
+        verbose: true
@@ -1,5 +1,61 @@
 # Change Log
 
+## v1.1.2 (2024-10-29)
+
+Bug fixes:
+
+- MQTT connection fixes:
+  - Retry MQTT topic subscription when it fails. When FWE starts up, it tries to establish the MQTT
+    connection and retries until it succeeds. Only after that, it subscribes to the topics. But if a
+    subscription failed (e.g. due to network issues), it never retried, making FWE never receive
+    messages from the topic until the process is restarted.
+  - On shutdown only unsubscribe to MQTT Channels that are subscribed
+  - When persistent sessions are enabled, don't unsubscribe on shutdown. This is required for FWE to
+    receive messages while it was offline.
+  - Connect only after all listeners are subscribed. If a message was received right after
+    connection was established, it could be silently ignored because the listeners for the topic
+    weren't registered yet.
+- ROS2 related bug fixes:
+  - If more than one campaign used a ROS2 signal in its expression, only the first campaign would
+    receive data for evaluation.
+  - If FWE receives a campaign more than once from the cloud, data collection would stop due to
+    internal signal IDs being reallocated.
+  - Fixed segfault for VSD build when campaigns received before decoder manifest
+- Fixed invalid read on shutdown when network is down
+
+Improvements:
+
+- Update AWS C++ SDK to `v1.11.284`.
+- Update Boost to `1.84.0`.
+- Enable flow control for MQTT5 client (when using `iotCore` connection type). This limits data sent
+  by FWE to what is defined in IoT Core limits for throughput and number of publishes.
+- Change default values for some MQTT connection settings and make them configurable in the config
+  file. All defaults are now the same as the AWS SDK:
+  - Keep alive interval changed from `60` seconds to `1200` seconds. Set `keepAliveIntervalSeconds`
+    to override it.
+  - Ping timeout default changed from `3000` ms to `30000` ms. Set `pingTimeoutMs` to override it.
+  - Persistent sessions are now disabled by default. Set `sessionExpiryIntervalSeconds` to a
+    non-zero value to enable it.
+- Add implicit casting between numeric and Boolean data types in expression evaluation. E.g.
+  `1 + true` will equal `2`, and `false || 3` will equal `true`.
+- Add support for relative paths to the certificate, private key files and persistency directory,
+  relative to the directory containing the configuration JSON file.
+- Make `demo.sh` more generic, by 1/ Allowing multiple 'node', 'decoder', 'network interface', and
+  'campaign' JSON files to be passed, rather than having specific options for CAN, OBD and ROS2, 2/
+  Add `--data-destination` option to specify data destination (default is still Amazon Timestream.)
+  This allows the Android-specific demo script to be removed.
+- Previously if the CAN network interface goes down, FWE would exit with an error. Now it will
+  continue to run and will resume data collection if the interface comes back up, however it will
+  still exit with an error if the interface is removed from the system.
+- Improved developer guides
+- Added Disconnect Packet to the MQTT Client Stop sequence to help customers understand what the
+  reason for the vehicle disconnection from the broker
+- Added usage of data type from decoder manifest for CAN and OBD signals
+- Add new signal type UNKNOWN
+- Split MQTT channels into separate Sender and Receiver compared to previously using the same
+  channel instance to send and receive messages
+- Adaptive payload sizing for both compressed and uncompressed data
+
 ## v1.1.1 (2024-02-12)
 
 Bug fixes:

@@ -2,7 +2,7 @@
 
 cmake_minimum_required(VERSION 3.10.2)
 
-project(iotfleetwise VERSION 1.1.1)
+project(iotfleetwise VERSION 1.1.2)
 
 # FWE uses C++14 for compatibility reasons with Automotive middlewares (Adaptive AUTOSAR, ROS2)
 # Note: When built with FWE_FEATURE_ROS2, colcon will override these settings
@@ -29,7 +29,8 @@ option(FWE_FEATURE_CUSTOM_DATA_SOURCE "Include the custom data source interface,
 option(FWE_FEATURE_IWAVE_GPS "Include the IWave GPS example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF)
 option(FWE_FEATURE_EXTERNAL_GPS "Include the external GPS example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF)
 option(FWE_FEATURE_AAOS_VHAL "Include the Android Automotive VHAL example for a custom data source (implies FWE_FEATURE_CUSTOM_DATA_SOURCE)" OFF)
-option(FWE_FEATURE_VISION_SYSTEM_DATA "Include support for vision-system-data sources" OFF)
+option(FWE_FEATURE_S3 "Internal option, not useful on its own. Enables usage of AWS credential provider, and uploading and downloading from S3." OFF)
+option(FWE_FEATURE_VISION_SYSTEM_DATA "Include support for vision-system-data sources. Implies FWE_FEATURE_S3." OFF)
 option(FWE_FEATURE_ROS2 "Include support for ROS2 as a vision-system-data source. Implies FWE_FEATURE_VISION_SYSTEM_DATA." OFF)
 option(FWE_BUILD_EXECUTABLE "Build the executable, otherwise build a library" ON)
 option(FWE_BUILD_ANDROID_SHARED_LIBRARY "Build the android shared library" OFF)
@@ -57,8 +58,12 @@ if(FWE_FEATURE_ROS2)
   ament_package()
 endif()
 if(FWE_FEATURE_VISION_SYSTEM_DATA)
+  set(FWE_FEATURE_S3 ON FORCE)
   add_compile_options("-DFWE_FEATURE_VISION_SYSTEM_DATA;-DDECNUMDIGITS=34")
 endif()
+if(FWE_FEATURE_S3)
+  add_compile_options("-DFWE_FEATURE_S3")
+endif()
 
 # Define the default build type
 if(NOT CMAKE_BUILD_TYPE)
@@ -116,15 +121,17 @@ set_source_files_properties(${PROTO_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-convers
 set(HEADER_FILES
   src/Assert.h
   src/AwsBootstrap.h
-  src/AwsIotChannel.h
   src/AwsIotConnectivityModule.h
+  src/AwsIotReceiver.h
+  src/AwsIotSender.h
   src/AwsSDKMemoryManager.h
   src/CacheAndPersist.h
   src/CANDataConsumer.h
   src/CANDataSource.h
   src/CANDataTypes.h
   src/CANDecoder.h
   src/CANInterfaceIDTranslator.h
+  src/CheckinSender.h
   src/Clock.h
   src/ClockHandler.h
   src/CollectionInspectionAPITypes.h
@@ -138,15 +145,14 @@ set(HEADER_FILES
   src/DataSenderManager.h
   src/DataSenderManagerWorkerThread.h
   src/DataSenderProtoWriter.h
+  src/DataSenderTypes.h
   src/DecoderManifestIngestion.h
   src/EnumUtility.h
   src/EventTypes.h
   src/ExternalCANDataSource.h
   src/ICollectionScheme.h
   src/ICollectionSchemeList.h
-  src/ICollectionSchemeManager.h
   src/IConnectionTypes.h
-  src/IConnectivityChannel.h
   src/IConnectivityModule.h
   src/IDecoderDictionary.h
   src/IDecoderManifest.h
@@ -161,6 +167,7 @@ set(HEADER_FILES
   src/ISOTPOverCANSender.h
   src/ISOTPOverCANSenderReceiver.h
   src/Listener.h
+  src/QueueTypes.h
   src/LoggingModule.h
   src/LogLevel.h
   src/MemoryUsageInfo.h
@@ -171,13 +178,16 @@ set(HEADER_FILES
   src/OBDOverCANECU.h
   src/OBDOverCANModule.h
   src/PayloadManager.h
+  src/QueueTypes.h
+  src/RawDataManager.h
   src/RemoteProfiler.h
   src/RetryThread.h
   src/Schema.h
   src/SchemaListener.h
   src/Signal.h
   src/SignalTypes.h
   src/StreambufBuilder.h
+  src/TelemetryDataSender.h
   src/Thread.h
   src/Timer.h
   src/TimeTypes.h
@@ -188,14 +198,15 @@ set(HEADER_FILES
 # Source files
 set(SRC_FILES
   src/AwsBootstrap.cpp
-  src/AwsIotChannel.cpp
+  src/AwsIotReceiver.cpp
+  src/AwsIotSender.cpp
   src/AwsIotConnectivityModule.cpp
   src/AwsSDKMemoryManager.cpp
   src/CacheAndPersist.cpp
   src/CANDataConsumer.cpp
   src/CANDataSource.cpp
   src/CANDecoder.cpp
-  src/CheckinAndPersistency.cpp
+  src/CheckinSender.cpp
   src/ClockHandler.cpp
   src/CollectionInspectionEngine.cpp
   src/CollectionInspectionWorkerThread.cpp
@@ -222,9 +233,12 @@ set(SRC_FILES
   src/OBDOverCANECU.cpp
   src/OBDOverCANModule.cpp
   src/PayloadManager.cpp
+  src/Persistency.cpp
+  src/RawDataManager.cpp
   src/RemoteProfiler.cpp
   src/RetryThread.cpp
   src/Schema.cpp
+  src/TelemetryDataSender.cpp
   src/Thread.cpp
   src/TraceModule.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/IoTFleetWiseVersion.cpp
@@ -237,7 +251,6 @@ set(TEST_FILES
   test/unit/CacheAndPersistTest.cpp
   test/unit/CANDataSourceTest.cpp
   test/unit/CANDecoderTest.cpp
-  test/unit/CheckinAndPersistencyTest.cpp
   test/unit/ClockHandlerTest.cpp
   test/unit/CollectionInspectionEngineTest.cpp
   test/unit/CollectionInspectionWorkerThreadTest.cpp
@@ -258,6 +271,8 @@ set(TEST_FILES
   test/unit/OBDDataDecoderTest.cpp
   test/unit/OBDOverCANModuleTest.cpp
   test/unit/PayloadManagerTest.cpp
+  test/unit/PersistencyTest.cpp
+  test/unit/RawDataManagerTest.cpp
   test/unit/RemoteProfilerTest.cpp
   test/unit/SchemaTest.cpp
   test/unit/ThreadTest.cpp
@@ -292,28 +307,39 @@ if(FWE_FEATURE_CUSTOM_DATA_SOURCE)
   set(HEADER_FILES ${HEADER_FILES} src/CustomDataSource.h)
 endif()
 if(FWE_FEATURE_GREENGRASSV2)
-  set(SRC_FILES ${SRC_FILES} src/AwsGGConnectivityModule.cpp src/AwsGGChannel.cpp)
-  set(HEADER_FILES ${HEADER_FILES} src/AwsGGChannel.h src/AwsGGConnectivityModule.h)
+  set(SRC_FILES ${SRC_FILES}
+    src/AwsGreengrassV2ConnectivityModule.cpp
+    src/AwsGreengrassV2Receiver.cpp
+    src/AwsGreengrassV2Sender.cpp
+  )
+  set(HEADER_FILES ${HEADER_FILES}
+    src/AwsGreengrassV2ConnectivityModule.h
+    src/AwsGreengrassV2Receiver.h
+    src/AwsGreengrassV2Sender.h
+)
+endif()
+if(FWE_FEATURE_S3)
+  set(SRC_FILES ${SRC_FILES} src/Credentials.cpp)
+  set(TEST_FILES ${TEST_FILES} test/unit/CredentialsTest.cpp)
+  set(HEADER_FILES ${HEADER_FILES}
+    src/Credentials.h
+    src/TransferManagerWrapper.h
+  )
 endif()
 if(FWE_FEATURE_VISION_SYSTEM_DATA)
   set(SRC_FILES ${SRC_FILES}
-    src/Credentials.cpp
     src/DataSenderIonWriter.cpp
-    src/RawDataManager.cpp
     src/S3Sender.cpp
+    src/VisionSystemDataSender.cpp
   )
   set(TEST_FILES ${TEST_FILES}
-    test/unit/CredentialsTest.cpp
     test/unit/DataSenderIonWriterTest.cpp
-    test/unit/RawDataManagerTest.cpp
     test/unit/S3SenderTest.cpp
   )
   set(HEADER_FILES ${HEADER_FILES}
-    src/Credentials.h
     src/DataSenderIonWriter.h
-    src/RawDataManager.h
     src/S3Sender.h
-    src/TransferManagerWrapper.h
+    src/VisionSystemDataSender.h
   )
 endif()
 if(FWE_FEATURE_ROS2)
@@ -357,7 +383,7 @@ endif()
 separate_arguments(FWE_AWS_SDK_EXTRA_LIBS)
 
 set(REQUIRED_AWS_SDK_COMPONENTS "core")
-if(FWE_FEATURE_VISION_SYSTEM_DATA)
+if(FWE_FEATURE_S3)
   set(REQUIRED_AWS_SDK_COMPONENTS "transfer;s3-crt;${REQUIRED_AWS_SDK_COMPONENTS}")
 endif()
 set(OLD_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
@@ -546,10 +572,10 @@ if(${BUILD_TESTING})
     add_executable(ROS2DataSourceTest
       test/unit/ROS2DataSourceTest.cpp
       src/ROS2DataSource.cpp
+      src/RawDataManager.cpp
       src/Thread.cpp
       src/LoggingModule.cpp
       src/ClockHandler.cpp
-      src/RawDataManager.cpp
       src/ConsoleLogger.cpp
       src/IoTFleetWiseConfig.cpp
       test/unit/support/main.cpp