feature: E2E tests additional auth methods, easier config, and pipeline yaml (#864)

* add other auth methods

* update host code to use environment

* add pipeline yaml

* rename device structure and fixture

Author: BertKleewein

device_e2e/aio/conftest.py

@@ -8,6 +8,7 @@
 import e2e_settings
 import test_config
 import logging
+from utils import create_client_object
 from service_helper import ServiceHelper
 from azure.iot.device.iothub.aio import IoTHubDeviceClient, IoTHubModuleClient
 
@@ -23,31 +24,10 @@ def event_loop():
 
 
 @pytest.fixture(scope="function")
-async def brand_new_client(client_kwargs):
-    client = None
-
-    if test_config.config.identity == test_config.IDENTITY_DEVICE_CLIENT:
-        ClientClass = IoTHubDeviceClient
-    elif test_config.config.identity == test_config.IDENTITY_MODULE_CLIENT:
-        ClientClass = IoTHubModuleClient
-    else:
-        raise Exception("config.identity invalid")
-
-    if test_config.config.auth == test_config.AUTH_CONNECTION_STRING:
-        # TODO: This is currently using a connection string stored in _e2e_settings.xml.  This will move to be a dynamically created identity similar to the way node's device_identity_helper.js works.
-        logger.info(
-            "Creating {} using create_from_connection_string with kwargs={}".format(
-                ClientClass, client_kwargs
-            )
-        )
-        client = ClientClass.create_from_connection_string(
-            e2e_settings.DEVICE_CONNECTION_STRING, **client_kwargs
-        )
-    elif test_config.config.auth == test_config.X509:
-        # need to implement
-        raise Exception("X509 Auth not yet implemented")
-    else:
-        raise Exception("config.auth invalid")
+async def brand_new_client(device_identity, client_kwargs):
+    client = create_client_object(
+        device_identity, client_kwargs, IoTHubDeviceClient, IoTHubModuleClient
+    )
 
     yield client
 

device_e2e/aio/test_sas_renewal.py

@@ -13,8 +13,12 @@
 pytestmark = pytest.mark.asyncio
 
 
+@pytest.mark.skipif(
+    test_config.config.auth not in test_config.AUTH_WITH_RENEWING_TOKEN,
+    reason="{} auth does not support token renewal".format(test_config.config.auth),
+)
 @pytest.mark.describe("Client sas renewal code")
-class TestSasRenewalReconnectEnabled(object):
+class TestSasRenewal(object):
     @pytest.fixture(scope="class")
     def extra_client_kwargs(self):
         # should renew after 10 seconds

device_e2e/aio/test_send_message.py

@@ -44,6 +44,7 @@ def extra_client_kwargs(self):
         return {"keep_alive": 5}
 
     @pytest.mark.it("Sends if connection drops before sending")
+    @pytest.mark.uses_iptables
     async def test_sends_if_drop_before_sending(
         self, client, random_message, dropper, get_next_eventhub_arrival
     ):
@@ -74,6 +75,7 @@ async def test_sends_if_drop_before_sending(
         logger.info("Success")
 
     @pytest.mark.it("Sends if connection rejects send")
+    @pytest.mark.uses_iptables
     async def test_sends_if_reject_before_sending(
         self, client, random_message, dropper, get_next_eventhub_arrival
     ):
@@ -137,6 +139,7 @@ async def test_connect_if_necessary(self, client, random_message, get_next_event
         assert json.dumps(event.message_body) == random_message.data
 
     @pytest.mark.it("Automatically connects if transport automatically disconnected before sending")
+    @pytest.mark.uses_iptables
     async def test_connects_after_automatic_disconnect(
         self, client, random_message, dropper, get_next_eventhub_arrival
     ):
@@ -156,6 +159,7 @@ async def test_connects_after_automatic_disconnect(
         assert json.dumps(event.message_body) == random_message.data
 
     @pytest.mark.it("Fails if connection disconnects before sending")
+    @pytest.mark.uses_iptables
     async def test_fails_if_disconnect_before_sending(self, client, random_message, dropper):
 
         assert client.connected
@@ -170,6 +174,7 @@ async def test_fails_if_disconnect_before_sending(self, client, random_message,
             await send_task
 
     @pytest.mark.it("Fails if connection drops before sending")
+    @pytest.mark.uses_iptables
     async def test_fails_if_drop_before_sending(self, client, random_message, dropper):
 
         assert client.connected

device_e2e/aio/test_twin.py

@@ -21,29 +21,35 @@
 @pytest.mark.describe("Device Client Reported Properties")
 class TestReportedProperties(object):
     @pytest.mark.it("Can set a simple reported property")
-    async def test_simple_patch(self, client, reported_props, get_next_reported_patch_arrival):
+    async def test_simple_patch(
+        self, client, random_reported_props, get_next_reported_patch_arrival
+    ):
 
         # patch properties
-        await client.patch_twin_reported_properties(reported_props)
+        await client.patch_twin_reported_properties(random_reported_props)
 
         # wait for patch to arrive at service and verify
         received_patch = await get_next_reported_patch_arrival()
         assert (
-            received_patch[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+            received_patch[const.REPORTED][const.TEST_CONTENT]
+            == random_reported_props[const.TEST_CONTENT]
         )
 
         # get twin from the service and verify content
         twin = await client.get_twin()
-        assert twin[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+        assert twin[const.REPORTED][const.TEST_CONTENT] == random_reported_props[const.TEST_CONTENT]
 
     @pytest.mark.it("Can clear a reported property")
-    async def test_clear_property(self, client, reported_props, get_next_reported_patch_arrival):
+    async def test_clear_property(
+        self, client, random_reported_props, get_next_reported_patch_arrival
+    ):
 
         # patch properties and verify that the service received the patch
-        await client.patch_twin_reported_properties(reported_props)
+        await client.patch_twin_reported_properties(random_reported_props)
         received_patch = await get_next_reported_patch_arrival()
         assert (
-            received_patch[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+            received_patch[const.REPORTED][const.TEST_CONTENT]
+            == random_reported_props[const.TEST_CONTENT]
         )
 
         # send a patch clearing properties and verify that the service received that patch
@@ -60,22 +66,23 @@ async def test_clear_property(self, client, reported_props, get_next_reported_pa
 
     @pytest.mark.it("Connects the transport if necessary")
     async def test_connect_if_necessary(
-        self, client, reported_props, get_next_reported_patch_arrival
+        self, client, random_reported_props, get_next_reported_patch_arrival
     ):
 
         await client.disconnect()
 
         assert not client.connected
-        await client.patch_twin_reported_properties(reported_props)
+        await client.patch_twin_reported_properties(random_reported_props)
         assert client.connected
 
         received_patch = await get_next_reported_patch_arrival()
         assert (
-            received_patch[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+            received_patch[const.REPORTED][const.TEST_CONTENT]
+            == random_reported_props[const.TEST_CONTENT]
         )
 
         twin = await client.get_twin()
-        assert twin[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+        assert twin[const.REPORTED][const.TEST_CONTENT] == random_reported_props[const.TEST_CONTENT]
 
 
 @pytest.mark.dropped_connection
@@ -89,13 +96,15 @@ def extra_client_kwargs(self):
 
     @pytest.mark.it("Sends if connection drops before sending")
     async def test_sends_if_drop_before_sending(
-        self, client, reported_props, dropper, get_next_reported_patch_arrival
+        self, client, random_reported_props, dropper, get_next_reported_patch_arrival
     ):
 
         assert client.connected
         dropper.drop_outgoing()
 
-        send_task = asyncio.create_task(client.patch_twin_reported_properties(reported_props))
+        send_task = asyncio.create_task(
+            client.patch_twin_reported_properties(random_reported_props)
+        )
         while client.connected:
             await asyncio.sleep(1)
 
@@ -109,18 +118,21 @@ async def test_sends_if_drop_before_sending(
 
         received_patch = await get_next_reported_patch_arrival()
         assert (
-            received_patch[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+            received_patch[const.REPORTED][const.TEST_CONTENT]
+            == random_reported_props[const.TEST_CONTENT]
         )
 
     @pytest.mark.it("Sends if connection rejects send")
     async def test_sends_if_reject_before_sending(
-        self, client, reported_props, dropper, get_next_reported_patch_arrival
+        self, client, random_reported_props, dropper, get_next_reported_patch_arrival
     ):
 
         assert client.connected
         dropper.reject_outgoing()
 
-        send_task = asyncio.create_task(client.patch_twin_reported_properties(reported_props))
+        send_task = asyncio.create_task(
+            client.patch_twin_reported_properties(random_reported_props)
+        )
         while client.connected:
             await asyncio.sleep(1)
 
@@ -134,7 +146,8 @@ async def test_sends_if_reject_before_sending(
 
         received_patch = await get_next_reported_patch_arrival()
         assert (
-            received_patch[const.REPORTED][const.TEST_CONTENT] == reported_props[const.TEST_CONTENT]
+            received_patch[const.REPORTED][const.TEST_CONTENT]
+            == random_reported_props[const.TEST_CONTENT]
         )
 
 

device_e2e/client_fixtures.py

@@ -8,11 +8,7 @@
 import functools
 import time
 import sys
-import const
 import test_config
-from azure.iot.device.iothub import Message
-from utils import get_random_message, get_random_dict
-from azure.iot.device.iothub import IoTHubDeviceClient
 
 
 @pytest.fixture(scope="function")
@@ -26,23 +22,13 @@ def module_id(brand_new_client):
     return brand_new_client._mqtt_pipeline._pipeline.pipeline_configuration.module_id
 
 
-@pytest.fixture(scope="function")
-def reported_props():
-    return {const.TEST_CONTENT: get_random_dict()}
-
-
 @pytest.fixture(scope="function")
 def watches_events(service_helper, device_id, module_id):
     service_helper.start_watching(device_id, module_id)
     yield
     service_helper.stop_watching(device_id, module_id)
 
 
-@pytest.fixture(scope="function")
-def random_message():
-    return get_random_message()
-
-
 @pytest.fixture(scope="function")
 def connection_retry():
     return True

device_e2e/conftest.py

@@ -5,6 +5,10 @@
 import logging
 import concurrent.futures
 import test_config
+import device_identity_helper
+import const
+import sys
+from utils import get_random_message, get_random_dict, is_windows
 
 # noqa: F401 defined in .flake8 file in root of repo
 
@@ -27,15 +31,16 @@
     websockets,
     device_id,
     module_id,
-    reported_props,
     watches_events,
-    random_message,
 )
 
 logging.basicConfig(level=logging.WARNING)
 logging.getLogger("e2e").setLevel(level=logging.DEBUG)
 logging.getLogger("paho").setLevel(level=logging.DEBUG)
-logging.getLogger("azure.iot").setLevel(level=logging.DEBUG)
+logging.getLogger("azure.iot").setLevel(level=logging.INFO)
+
+logger = logging.getLogger(__name__)
+logger.setLevel(level=logging.INFO)
 
 
 @pytest.fixture(scope="module")
@@ -48,6 +53,44 @@ def executor():
     return concurrent.futures.ThreadPoolExecutor()
 
 
+@pytest.fixture(scope="function")
+def random_message():
+    return get_random_message()
+
+
+@pytest.fixture(scope="function")
+def random_reported_props():
+    return {const.TEST_CONTENT: get_random_dict()}
+
+
+@pytest.fixture(scope="session")
+def device_identity():
+
+    if test_config.config.auth == test_config.AUTH_CONNECTION_STRING:
+        device_identity = device_identity_helper.create_device_with_symmetric_key()
+        logger.info(
+            "Created connection string device with deviceId = {}".format(device_identity.device_id)
+        )
+    elif test_config.config.auth == test_config.AUTH_SYMMETRIC_KEY:
+        device_identity = device_identity_helper.create_device_with_symmetric_key()
+        logger.info(
+            "Created symmetric key device with deviceId = {}".format(device_identity.device_id)
+        )
+    elif test_config.config.auth == test_config.AUTH_SAS_TOKEN:
+        device_identity = device_identity_helper.create_device_with_sas()
+        logger.info("Created sas token device with deviceId = {}".format(device_identity.device_id))
+    elif test_config.config.auth in test_config.AUTH_CHOICES:
+        # need to implement
+        raise Exception("{} Auth not yet implemented".format(test_config.config.auth))
+    else:
+        raise Exception("config.auth invalid")
+
+    yield device_identity
+
+    logger.info("Deleting device with deviceId = {}".format(device_identity.device_id))
+    device_identity_helper.delete_device(device_identity.device_id)
+
+
 def pytest_addoption(parser):
     parser.addoption(
         "--transport",
@@ -68,11 +111,26 @@ def pytest_addoption(parser):
         help="Identity (client type) to use for tests",
         type=str,
         choices=test_config.IDENTITY_CHOICES,
-        default=test_config.IDENTITY_DEVICE_CLIENT,
+        default=test_config.IDENTITY_DEVICE,
     )
 
 
 def pytest_configure(config):
     test_config.config.transport = config.getoption("transport")
     test_config.config.auth = config.getoption("auth")
     test_config.config.identity = config.getoption("identity")
+
+
+def pytest_runtest_setup(item):
+    # tests that use iptables need to be skipped on Windows
+    if is_windows():
+        for x in item.iter_markers("uses_iptables"):
+            pytest.skip("test uses iptables")
+            break
+
+
+collect_ignore = ["test_settings.py"]
+
+# Ignore Async tests if below Python 3.5
+if sys.version_info < (3, 5):
+    collect_ignore.append("aio")