From 02a890fa1176686307129ceb7f1dd6e7b2c696b4 Mon Sep 17 00:00:00 2001 From: Sid Janga Date: Thu, 20 Feb 2020 13:36:24 -0800 Subject: [PATCH] Updated project file and added GTXResult API --- Classes/GTXArtifactProcessor.m | 2 + Classes/GTXChecksCollection.m | 4 +- Classes/GTXResult.h | 64 +++++++++++ Classes/GTXResult.m | 83 ++++++++++++++ Classes/GTXTestEnvironment.m | 78 +++++-------- Classes/GTXToolKit.h | 15 ++- Classes/GTXToolKit.m | 132 +++++++++++----------- Classes/GTXiLibCore.m | 23 ++-- GTXiLib.xcodeproj/project.pbxproj | 68 +++++++++++ Tests/UnitTests/GTXAnalyticsTests.m | 126 +++++++++++++++------ Tests/UnitTests/GTXBaseTestCase.h | 3 +- Tests/UnitTests/GTXBaseTestCase.m | 2 +- Tests/UnitTests/GTXResultTests.m | 65 +++++++++++ Tests/UnitTests/GTXToolKitTests.m | 167 ++++++++++++++++++---------- 14 files changed, 607 insertions(+), 225 deletions(-) create mode 100644 Classes/GTXResult.h create mode 100644 Classes/GTXResult.m create mode 100644 Tests/UnitTests/GTXResultTests.m diff --git a/Classes/GTXArtifactProcessor.m b/Classes/GTXArtifactProcessor.m index 9c3dd73..1db9ccd 100644 --- a/Classes/GTXArtifactProcessor.m +++ b/Classes/GTXArtifactProcessor.m @@ -87,6 +87,8 @@ - (void)fetchLatestReportAsync:(GTXArtifactProcessorReportFetchBlock)fetchBlock - (void)gtx_assertCallerIsCommandQueue { void *context = dispatch_queue_get_specific(_commandQueue, _queueKey); + (void)context; // Marking context as used when below assert is removed due to compiler + // optimizations. GTX_ASSERT(context == _queueContext, @"This method can only be executed from command queue."); } diff --git a/Classes/GTXChecksCollection.m b/Classes/GTXChecksCollection.m index 6f1746e..1fcedd8 100644 --- a/Classes/GTXChecksCollection.m +++ b/Classes/GTXChecksCollection.m @@ -572,15 +572,13 @@ + (nullable NSString *)gtx_errorDescriptionForMinimumTappableArea:(CGRect)frame /** * Determines if @c frame satisfies minimum touch target guidelines, and if it doesn't, adds an - * error description to @c array. + * error description to @c array. * * @param frame The frame of an interactable element. This can be its frame, accessibilityFrame, or * some other bounding box. * @param propertyName The name of the element's property being checked. This is included in the * description. * @param array The array to which to add the error description, if it exists. - * @return A string describing the error, or @c nil if @c frame satisfies minimum touch target - * guidelines. */ + (void)gtx_errorDescriptionForMinimumTappableArea:(CGRect)frame propertyName:(NSString *)propertyName diff --git a/Classes/GTXResult.h b/Classes/GTXResult.h new file mode 100644 index 0000000..ad91c38 --- /dev/null +++ b/Classes/GTXResult.h @@ -0,0 +1,64 @@ +// +// Copyright 2020 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + GTXResult encapsulates results (and any other associated info) of a single pass of accessibility + checks. + */ +@interface GTXResult : NSObject + +/** + An array of all the errors encountered during the accessibility check(s). +*/ +@property(nonatomic, strong, readonly) NSArray *errorsFound; + +/** + Total number of elements that passed through accessibilty check(s). +*/ +@property(nonatomic, assign, readonly) NSInteger elementsScanned; + +/** + Use -initWithErrorsFound:elementsScanned: +*/ +- (instancetype)init NS_UNAVAILABLE; + +/** + Initializes a new GTXResult object. + + @param errorsFound Array of errors found. + @param elementsScanned Total number of elements scanned. + @return A newly created GTXResult object. +*/ +- (instancetype)initWithErrorsFound:(NSArray *)errorsFound + elementsScanned:(NSInteger)elementsScanned; + +/** + @return @YES if none of the checks failed, @NO otherwise. +*/ +- (BOOL)allChecksPassed; + +/** + @return The aggregated error of all the errors in this result, or @c nil if no errors were found. +*/ +- (NSError *)aggregatedError; + +@end + +NS_ASSUME_NONNULL_END diff --git a/Classes/GTXResult.m b/Classes/GTXResult.m new file mode 100644 index 0000000..f13a5da --- /dev/null +++ b/Classes/GTXResult.m @@ -0,0 +1,83 @@ +// +// Copyright 2020 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#import "GTXResult.h" +#import "NSError+GTXAdditions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface GTXResult () + +@property(nonatomic, strong) NSArray *errorsFound; +@property(nonatomic, assign) NSInteger elementsScanned; + +@end + +@implementation GTXResult { + NSError *_aggregatedError; +} + +- (instancetype)initWithErrorsFound:(NSArray *)errorsFound + elementsScanned:(NSInteger)elementsScanned { + self = [super init]; + if (self) { + _errorsFound = errorsFound; + _elementsScanned = elementsScanned; + } + return self; +} + +- (BOOL)allChecksPassed { + return self.errorsFound.count == 0; +} + +- (NSError *)aggregatedError { + if (![self allChecksPassed]) { + // Combine the error descriptions from all the errors into the following format ('.' is an + // indent): + // . . Element: + // . . . . Error: + // . . . . Error: + // . . Element: + // . . . . Error: + // . . . . Error: + + NSMutableString *errorString = + [[NSMutableString alloc] initWithString:@"One or more elements FAILED the accessibility " + @"checks:\n"]; + for (NSError *error in self.errorsFound) { + id element = [[error userInfo] objectForKey:kGTXErrorFailingElementKey]; + if (element) { + // Add element description with an indent. + [errorString appendFormat:@" %@\n", element]; + for (NSError *underlyingError in + // Add element's error description with twice the indent. + [[error userInfo] objectForKey:kGTXErrorUnderlyingErrorsKey]) { + [errorString appendFormat:@" + %@\n", [underlyingError localizedDescription]]; + } + } + } + [NSError gtx_logOrSetError:&_aggregatedError + description:errorString + code:GTXCheckErrorCodeGenericError + userInfo:@{kGTXErrorUnderlyingErrorsKey : self.errorsFound}]; + } + return _aggregatedError; +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/Classes/GTXTestEnvironment.m b/Classes/GTXTestEnvironment.m index 40444ad..f953855 100644 --- a/Classes/GTXTestEnvironment.m +++ b/Classes/GTXTestEnvironment.m @@ -51,13 +51,18 @@ - (void)setAccessibilityPreferenceAsMobile:(CFStringRef)key /** * Path to accessibility utils framework on the simulator. */ -static NSString *const kPathToAXUtils = +static NSString *const kGTXPathToAXUtils = @"/System/Library/PrivateFrameworks/AccessibilityUtilities.framework/AccessibilityUtilities"; /** - * Class name of the private class: XCAXClient_iOS class. + * Path to accessibility dylib on the device. */ -static NSString *const kXCAXClientClassName = @"XCAXClient_iOS"; +static NSString *const kGTXPathToAXDyLib = @"/usr/lib/libAccessibility.dylib"; + +/** + * Name of the method that can enable accessibility. + */ +static char *const kGTXAXSetterMethodName = "_AXSSetAutomationEnabled"; #pragma mark - Implementations @@ -123,7 +128,7 @@ + (BOOL)_setAccessibilityPreference:(CFStringRef)key static AXBackBoardServer *server; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ - char const *const localPath = [kPathToAXUtils fileSystemRepresentation]; + char const *const localPath = [kGTXPathToAXUtils fileSystemRepresentation]; void *handle = dlopen(localPath, RTLD_LOCAL); if (!handle) { NSString *description = @@ -163,57 +168,32 @@ + (BOOL)_setAccessibilityPreference:(CFStringRef)key * Enables accessibility to allow using accessibility properties on devices. */ + (BOOL)gtx_enableAccessibilityOnDeviceWithError:(GTXErrorRefType)errorOrNil { - Class XCAXClientClass = NSClassFromString(kXCAXClientClassName); - if (XCAXClientClass == nil) { - NSString *description = [NSString stringWithFormat:@"%@ class not found", kXCAXClientClassName]; - [NSError gtx_logOrSetError:errorOrNil - description:description - code:GTXCheckErrorCodeInvalidTestEnvironment - userInfo:nil]; - return NO; - } - id XCAXClient = [XCAXClientClass sharedClient]; - if (XCAXClient == nil) { + GTX_LOG(@"Enabling accessibility to access UI accessibility properties."); + char const *const libAccessibilityPath = [kGTXPathToAXDyLib fileSystemRepresentation]; + void *handle = dlopen(libAccessibilityPath, RTLD_LOCAL); + + if (handle) { + void (*AXSetterMethod)(BOOL) = dlsym(handle, kGTXAXSetterMethodName); + if (AXSetterMethod) { + AXSetterMethod(YES); + return YES; + } else { + NSString *description = [NSString + stringWithFormat:@"Pointer to %s method must not be NULL", kGTXAXSetterMethodName]; + [NSError gtx_logOrSetError:errorOrNil + description:description + code:GTXCheckErrorCodeInvalidTestEnvironment + userInfo:nil]; + } + } else { NSString *description = - [NSString stringWithFormat:@"%@ sharedClient doesn't exist", kXCAXClientClassName]; + [NSString stringWithFormat:@"dlopen couldn't open libAccessibility.dylib at path %s", + libAccessibilityPath]; [NSError gtx_logOrSetError:errorOrNil description:description code:GTXCheckErrorCodeInvalidTestEnvironment userInfo:nil]; - return NO; } - // The method may not be available on versions older than iOS 9.1 - if ([XCAXClient respondsToSelector:@selector(loadAccessibility:)]) { - typedef void (*MethodType)(id, SEL, void*); - SEL selector = @selector(loadAccessibility:); - static void *unused = 0; - MethodType method = (MethodType)[XCAXClient methodForSelector:selector]; - method(XCAXClient, selector, &unused); - } else { - [NSError gtx_logOrSetError:errorOrNil - description:@"Could not enable accessibility! iOS version must be >= 9.1" - code:GTXCheckErrorCodeInvalidTestEnvironment - userInfo:nil]; - return NO; - } - return YES; -} - -#pragma mark - unused methods - -/** - * Unused method only used for selector name. - */ -- (id)sharedClient { - NSAssert(NO, @"This method must not be invoked directly, its only used for exposing selector"); - return nil; -} - -/** - * Unused method only used for selector name. - */ -- (BOOL)loadAccessibility:(void **)unused { - NSAssert(NO, @"This method must not be invoked directly, its only used for exposing selector"); return NO; } diff --git a/Classes/GTXToolKit.h b/Classes/GTXToolKit.h index 84e60cb..f486d54 100644 --- a/Classes/GTXToolKit.h +++ b/Classes/GTXToolKit.h @@ -20,6 +20,7 @@ #import "GTXBlacklisting.h" #import "GTXCheckBlock.h" #import "GTXChecking.h" +#import "GTXResult.h" NS_ASSUME_NONNULL_BEGIN @@ -83,6 +84,8 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)checkElement:(id)element error:(GTXErrorRefType)errorOrNil; /** + @deprecated Use -resultFromCheckingAllElementsFromRootElements: instead. + Applies the registered checks on all elements in the accessibility tree under the given root elements while respecting blacklisted elements. @@ -90,8 +93,16 @@ NS_ASSUME_NONNULL_BEGIN @param errorOrNil Error object to be filled with error info on check failures. @return @c YES if all checks passed on all elements @c NO otherwise. */ -- (BOOL)checkAllElementsFromRootElements:(NSArray *)rootElements - error:(GTXErrorRefType)errorOrNil; +- (BOOL)checkAllElementsFromRootElements:(NSArray *)rootElements error:(GTXErrorRefType)errorOrNil; + +/** + Applies the registered checks on all elements in the accessibility tree under the given root + elements while respecting blacklisted elements. + + @param rootElements An array of root elements whose accessibility trees are to be checked. + @return A @c GTXResult object encpsulating the results. + */ +- (GTXResult *)resultFromCheckingAllElementsFromRootElements:(NSArray *)rootElements; @end diff --git a/Classes/GTXToolKit.m b/Classes/GTXToolKit.m index b8e9d35..55ad3a8 100644 --- a/Classes/GTXToolKit.m +++ b/Classes/GTXToolKit.m @@ -21,6 +21,7 @@ #import "GTXBlacklistBlock.h" #import "GTXBlacklistFactory.h" #import "GTXChecksCollection.h" +#import "GTXLogging.h" #import "NSError+GTXAdditions.h" #pragma mark - Extension @@ -75,7 +76,7 @@ - (void)registerCheck:(id)check { for (id existingCheck in _checks) { NSAssert(![[existingCheck name] isEqualToString:[check name]], @"Check named %@ already exists!", [check name]); - (void)existingCheck; // Ensures 'existingCheck' is marked as used even if NSAssert is removed. + (void)existingCheck; // Ensures 'existingCheck' is marked as used even if NSAssert is removed. } [_checks addObject:check]; } @@ -85,70 +86,61 @@ - (void)registerBlacklist:(id)blacklist { } - (BOOL)checkElement:(id)element error:(GTXErrorRefType)errorOrNil { - return [self _checkElement:element analyticsEnabled:YES error:errorOrNil]; + return [self gtx_checkElement:element + analyticsEnabled:YES + outWasElementChecked:NULL + error:errorOrNil]; } -- (BOOL)checkAllElementsFromRootElements:(NSArray *)rootElements - error:(GTXErrorRefType)errorOrNil { +- (BOOL)checkAllElementsFromRootElements:(NSArray *)rootElements error:(GTXErrorRefType)errorOrNil { + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + GTX_LOG(@"-checkAllElementsFromRootElements:error: has been deprecated, please use " + @"-resultFromCheckingAllElementsFromRootElements: instead"); + }); + GTXResult *result = [self resultFromCheckingAllElementsFromRootElements:rootElements]; + if (errorOrNil) { + *errorOrNil = [result aggregatedError]; + } + return [result allChecksPassed]; +} + +- (GTXResult *)resultFromCheckingAllElementsFromRootElements:(NSArray *)rootElements { GTXAccessibilityTree *tree = [[GTXAccessibilityTree alloc] initWithRootElements:rootElements]; NSMutableArray *errors; + NSInteger elementsScanned = 0; // Check each element and collect all failures into the errors array. for (id element in tree) { NSError *error; - if (![self _checkElement:element analyticsEnabled:NO error:&error]) { + BOOL wasElementChecked; + if (![self gtx_checkElement:element + analyticsEnabled:NO + outWasElementChecked:&wasElementChecked + error:&error]) { if (!error) { NSString *genericErrorDescription = - @"One or more Gtx checks failed, error description was not provided by the check."; + @"One or more Gtx checks failed, error description was not provided by the check."; error = [NSError errorWithDomain:kGTXErrorDomain code:GTXCheckErrorCodeGenericError - userInfo:@{ kGTXErrorFailingElementKey : element, - NSLocalizedDescriptionKey : genericErrorDescription }]; + userInfo:@{ + kGTXErrorFailingElementKey : element, + NSLocalizedDescriptionKey : genericErrorDescription + }]; } if (!errors) { errors = [[NSMutableArray alloc] init]; } [errors addObject:error]; } + if (wasElementChecked) { + elementsScanned += 1; + } } - [GTXAnalytics invokeAnalyticsEvent:(errors ? - GTXAnalyticsEventChecksFailed : - GTXAnalyticsEventChecksPerformed)]; - if (errors) { - // Combine the error descriptions from all the errors into the following format ('.' is an - // indent): - // . . Element: - // . . . . Error: - // . . . . Error: - // . . Element: - // . . . . Error: - // . . . . Error: + [GTXAnalytics invokeAnalyticsEvent:(errors ? GTXAnalyticsEventChecksFailed + : GTXAnalyticsEventChecksPerformed)]; - NSMutableString *errorString = - [[NSMutableString alloc] initWithString:@"One or more elements FAILED the accessibility " - @"checks:\n"]; - for (NSError *error in errors) { - id element = [[error userInfo] objectForKey:kGTXErrorFailingElementKey]; - if (element) { - // Add element description with an indent. - [errorString appendFormat:@" %@\n", element]; - for (NSError *underlyingError in - // Add element's error description with twice the indent. - [[error userInfo] objectForKey:kGTXErrorUnderlyingErrorsKey]) { - [errorString appendFormat:@" + %@\n", [underlyingError localizedDescription]]; - } - } - } - NSError *error; - [NSError gtx_logOrSetError:&error - description:errorString - code:GTXCheckErrorCodeGenericError - userInfo:@{ kGTXErrorUnderlyingErrorsKey : errors }]; - if (errorOrNil) { - *errorOrNil = error; - } - } - return errors == nil; + return [[GTXResult alloc] initWithErrorsFound:errors elementsScanned:elementsScanned]; } #pragma mark - Private @@ -158,13 +150,19 @@ - (BOOL)checkAllElementsFromRootElements:(NSArray *)rootElements @param element element to be checked. @param checkAnalyticsEnabled Boolean that indicates if analytics events are to be invoked. + @param[out] outWasElementChecked Optional reference to a boolean that will be set to @c YES if + element was checked, @c NO otherwise. @param errorOrNil Error object to be filled with error info on check failures. @return @c YES if all checks passed @c NO otherwise. */ -- (BOOL)_checkElement:(id)element - analyticsEnabled:(BOOL)checkAnalyticsEnabled - error:(GTXErrorRefType)errorOrNil { +- (BOOL)gtx_checkElement:(id)element + analyticsEnabled:(BOOL)checkAnalyticsEnabled + outWasElementChecked:(nullable BOOL *)outWasElementChecked + error:(GTXErrorRefType)errorOrNil { NSParameterAssert(element); + if (outWasElementChecked) { + *outWasElementChecked = NO; + } if ([element respondsToSelector:@selector(isAccessibilityElement)] && ![element isAccessibilityElement]) { // Currently all checks are only applicable to accessibility elements. @@ -194,23 +192,28 @@ - (BOOL)_checkElement:(id)element if (!error) { // Check failed but an error description was not provided, generate a generic description. NSString *errorDescription = - [NSString stringWithFormat:@"Check \"%@\" failed, no description was provided by the " - @"check implementation.", [checker name]]; + [NSString stringWithFormat:@"Check \"%@\" failed, no description was provided by the " + @"check implementation.", + [checker name]]; error = [NSError errorWithDomain:kGTXErrorDomain code:GTXCheckErrorCodeAccessibilityCheckFailed - userInfo:@{ NSLocalizedDescriptionKey: errorDescription, - kGTXErrorFailingElementKey: element }]; + userInfo:@{ + NSLocalizedDescriptionKey : errorDescription, + kGTXErrorFailingElementKey : element + }]; } if (!failedCheckErrors) { failedCheckErrors = [[NSMutableArray alloc] init]; } [failedCheckErrors addObject:error]; } + if (outWasElementChecked) { + *outWasElementChecked = YES; + } } if (checkAnalyticsEnabled) { - [GTXAnalytics invokeAnalyticsEvent:(failedCheckErrors ? - GTXAnalyticsEventChecksFailed : - GTXAnalyticsEventChecksPerformed)]; + [GTXAnalytics invokeAnalyticsEvent:(failedCheckErrors ? GTXAnalyticsEventChecksFailed + : GTXAnalyticsEventChecksPerformed)]; } if (!failedCheckErrors) { // All checks passed. @@ -224,8 +227,10 @@ - (BOOL)_checkElement:(id)element [NSError gtx_logOrSetError:&error description:errorDescription code:GTXCheckErrorCodeAccessibilityCheckFailed - userInfo:@{ kGTXErrorFailingElementKey : element, - kGTXErrorUnderlyingErrorsKey : failedCheckErrors}]; + userInfo:@{ + kGTXErrorFailingElementKey : element, + kGTXErrorUnderlyingErrorsKey : failedCheckErrors + }]; if (errorOrNil) { *errorOrNil = error; @@ -242,17 +247,14 @@ - (BOOL)_checkElement:(id)element */ - (NSString *)gtx_errorDescriptionForElement:(id)element gtxCheckErrors:(NSArray *)errors { NSMutableString *localizedDescription = - [NSMutableString stringWithFormat:@"%d accessibility error(s) were found in %@:\n", - (int)[errors count], - element]; + [NSMutableString stringWithFormat:@"%d accessibility error(s) were found in %@:\n", + (int)[errors count], element]; for (NSUInteger index = 0; index < [errors count]; index++) { - [localizedDescription appendString: - [NSString stringWithFormat:@"%d. %@\n", - (int)(index + 1), - [errors[index] localizedDescription]]]; + [localizedDescription + appendString:[NSString stringWithFormat:@"%d. %@\n", (int)(index + 1), + [errors[index] localizedDescription]]]; } return localizedDescription; } @end - diff --git a/Classes/GTXiLibCore.m b/Classes/GTXiLibCore.m index f0a4efa..76eea82 100644 --- a/Classes/GTXiLibCore.m +++ b/Classes/GTXiLibCore.m @@ -35,9 +35,9 @@ @interface GTXInstallOptions : NSObject -@property (nonatomic, strong) NSArray *checks; -@property (nonatomic, strong) NSArray *elementBlacklist; -@property (nonatomic, strong) GTXTestSuite *suite; +@property(nonatomic, strong) NSArray *checks; +@property(nonatomic, strong) NSArray *elementBlacklist; +@property(nonatomic, strong) GTXTestSuite *suite; @end @@ -74,7 +74,6 @@ @implementation GTXInstallOptions */ static BOOL gIsInTearDown; - #pragma mark - Implementation @implementation GTXiLib @@ -98,7 +97,7 @@ + (void)installOnTestSuite:(GTXTestSuite *)suite NSAssert(intersection.tests.count == 0, @"Error! Attempting to install GTXChecks multiple times on the same test cases: %@", intersection); - (void)intersection; // Ensures 'intersection' is marked as used even if NSAssert is removed. + (void)intersection; // Ensures 'intersection' is marked as used even if NSAssert is removed. } [gIntsallOptions addObject:options]; @@ -130,8 +129,7 @@ + (void)setFailureHandler:(GTXiLibFailureHandler)handler { + (GTXiLibFailureHandler)failureHandler { return gFailureHandler ?: ^(NSError *error) { NSString *formattedError = - [NSString stringWithFormat:@"\n\n%@\n\n", - error.localizedDescription]; + [NSString stringWithFormat:@"\n\n%@\n\n", error.localizedDescription]; [[NSAssertionHandler currentHandler] handleFailureInMethod:_cmd object:self file:@(__FILE__) @@ -170,10 +168,10 @@ + (BOOL)gtx_checkElement:(id)element { @return @c NO if any of the checks failed, @c YES otherwise. */ + (BOOL)gtx_checkAllElementsFromRootElements:(NSArray *)rootElements { - NSError *error; - BOOL success = [gToolkit checkAllElementsFromRootElements:rootElements error:&error]; - if (error) { - self.failureHandler(error); + GTXResult *result = [gToolkit resultFromCheckingAllElementsFromRootElements:rootElements]; + BOOL success = [result allChecksPassed]; + if (!success) { + self.failureHandler([result aggregatedError]); } return success; } @@ -191,8 +189,7 @@ + (void)gtx_testCaseDidBegin:(NSNotification *)notification { ((NSInvocation *)notification.userInfo[gtxTestInvocationUserInfoKey]).selector; GTXInstallOptions *currentTestCaseOptions = nil; for (GTXInstallOptions *options in gIntsallOptions) { - if ([options.suite hasTestCaseWithClass:currentTestClass - testMethod:currentTestSelector]) { + if ([options.suite hasTestCaseWithClass:currentTestClass testMethod:currentTestSelector]) { currentTestCaseOptions = options; break; } diff --git a/GTXiLib.xcodeproj/project.pbxproj b/GTXiLib.xcodeproj/project.pbxproj index 6650126..5cb4b3e 100644 --- a/GTXiLib.xcodeproj/project.pbxproj +++ b/GTXiLib.xcodeproj/project.pbxproj @@ -43,6 +43,23 @@ 61284E05211E447E00F8FF29 /* GTXTestGtxCanDetectFailures.m in Sources */ = {isa = PBXBuildFile; fileRef = 61284DF8211E447D00F8FF29 /* GTXTestGtxCanDetectFailures.m */; }; 61284E0A211E45C600F8FF29 /* GTXBlacklistFactory.h in Headers */ = {isa = PBXBuildFile; fileRef = 61284E08211E45C500F8FF29 /* GTXBlacklistFactory.h */; settings = {ATTRIBUTES = (Public, ); }; }; 61284E0B211E45C600F8FF29 /* GTXBlacklistFactory.m in Sources */ = {isa = PBXBuildFile; fileRef = 61284E09211E45C600F8FF29 /* GTXBlacklistFactory.m */; }; + 6133087023FF2F39003F8D41 /* GTXArtifactCollector.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133086B23FF2F39003F8D41 /* GTXArtifactCollector.m */; }; + 6133087123FF2F39003F8D41 /* GTXArtifactProcessor.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133086C23FF2F39003F8D41 /* GTXArtifactProcessor.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133087223FF2F39003F8D41 /* GTXArtifactProcessor.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133086D23FF2F39003F8D41 /* GTXArtifactProcessor.m */; }; + 6133087323FF2F39003F8D41 /* GTXArtifactCollector.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133086E23FF2F39003F8D41 /* GTXArtifactCollector.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133087423FF2F39003F8D41 /* GTXArtifactImplementing.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133086F23FF2F39003F8D41 /* GTXArtifactImplementing.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088123FF2F53003F8D41 /* UIColor+GTXAdditions.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087523FF2F53003F8D41 /* UIColor+GTXAdditions.m */; }; + 6133088223FF2F53003F8D41 /* UIColor+GTXAdditions.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133087623FF2F53003F8D41 /* UIColor+GTXAdditions.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088323FF2F53003F8D41 /* GTXError.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133087723FF2F53003F8D41 /* GTXError.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088423FF2F53003F8D41 /* GTXSnapshotContainer.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087823FF2F53003F8D41 /* GTXSnapshotContainer.m */; }; + 6133088523FF2F53003F8D41 /* GTXSnapshotBuffer.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087923FF2F53003F8D41 /* GTXSnapshotBuffer.m */; }; + 6133088623FF2F53003F8D41 /* GTXError.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087A23FF2F53003F8D41 /* GTXError.m */; }; + 6133088723FF2F53003F8D41 /* GTXElementBlacklist.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087B23FF2F53003F8D41 /* GTXElementBlacklist.m */; }; + 6133088823FF2F53003F8D41 /* GTXReport.m in Sources */ = {isa = PBXBuildFile; fileRef = 6133087C23FF2F53003F8D41 /* GTXReport.m */; }; + 6133088923FF2F53003F8D41 /* GTXSnapshotBuffer.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133087D23FF2F53003F8D41 /* GTXSnapshotBuffer.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088A23FF2F53003F8D41 /* GTXElementBlacklist.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133087E23FF2F53003F8D41 /* GTXElementBlacklist.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088B23FF2F53003F8D41 /* GTXSnapshotContainer.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133087F23FF2F53003F8D41 /* GTXSnapshotContainer.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6133088C23FF2F53003F8D41 /* GTXReport.h in Headers */ = {isa = PBXBuildFile; fileRef = 6133088023FF2F53003F8D41 /* GTXReport.h */; settings = {ATTRIBUTES = (Public, ); }; }; 61A0C4DE2061896300DF0169 /* GTXiLibCore.m in Sources */ = {isa = PBXBuildFile; fileRef = 61A0C4DB2061896300DF0169 /* GTXiLibCore.m */; }; 61A0C4DF2061896300DF0169 /* GTXiLib.h in Headers */ = {isa = PBXBuildFile; fileRef = 61A0C4DC2061896300DF0169 /* GTXiLib.h */; settings = {ATTRIBUTES = (Public, ); }; }; 61A0C4E02061896300DF0169 /* GTXiLibCore.h in Headers */ = {isa = PBXBuildFile; fileRef = 61A0C4DD2061896300DF0169 /* GTXiLibCore.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -147,6 +164,23 @@ 61284DF8211E447D00F8FF29 /* GTXTestGtxCanDetectFailures.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXTestGtxCanDetectFailures.m; path = Tests/FunctionalTests/TestApp/TestSources/GTXTestGtxCanDetectFailures.m; sourceTree = SOURCE_ROOT; }; 61284E08211E45C500F8FF29 /* GTXBlacklistFactory.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXBlacklistFactory.h; path = Classes/GTXBlacklistFactory.h; sourceTree = SOURCE_ROOT; }; 61284E09211E45C600F8FF29 /* GTXBlacklistFactory.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXBlacklistFactory.m; path = Classes/GTXBlacklistFactory.m; sourceTree = SOURCE_ROOT; }; + 6133086B23FF2F39003F8D41 /* GTXArtifactCollector.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXArtifactCollector.m; path = Classes/GTXArtifactCollector.m; sourceTree = SOURCE_ROOT; }; + 6133086C23FF2F39003F8D41 /* GTXArtifactProcessor.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXArtifactProcessor.h; path = Classes/GTXArtifactProcessor.h; sourceTree = SOURCE_ROOT; }; + 6133086D23FF2F39003F8D41 /* GTXArtifactProcessor.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXArtifactProcessor.m; path = Classes/GTXArtifactProcessor.m; sourceTree = SOURCE_ROOT; }; + 6133086E23FF2F39003F8D41 /* GTXArtifactCollector.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXArtifactCollector.h; path = Classes/GTXArtifactCollector.h; sourceTree = SOURCE_ROOT; }; + 6133086F23FF2F39003F8D41 /* GTXArtifactImplementing.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXArtifactImplementing.h; path = Classes/GTXArtifactImplementing.h; sourceTree = SOURCE_ROOT; }; + 6133087523FF2F53003F8D41 /* UIColor+GTXAdditions.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = "UIColor+GTXAdditions.m"; path = "Classes/UIColor+GTXAdditions.m"; sourceTree = SOURCE_ROOT; }; + 6133087623FF2F53003F8D41 /* UIColor+GTXAdditions.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "UIColor+GTXAdditions.h"; path = "Classes/UIColor+GTXAdditions.h"; sourceTree = SOURCE_ROOT; }; + 6133087723FF2F53003F8D41 /* GTXError.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXError.h; path = Classes/GTXError.h; sourceTree = SOURCE_ROOT; }; + 6133087823FF2F53003F8D41 /* GTXSnapshotContainer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXSnapshotContainer.m; path = Classes/GTXSnapshotContainer.m; sourceTree = SOURCE_ROOT; }; + 6133087923FF2F53003F8D41 /* GTXSnapshotBuffer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXSnapshotBuffer.m; path = Classes/GTXSnapshotBuffer.m; sourceTree = SOURCE_ROOT; }; + 6133087A23FF2F53003F8D41 /* GTXError.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXError.m; path = Classes/GTXError.m; sourceTree = SOURCE_ROOT; }; + 6133087B23FF2F53003F8D41 /* GTXElementBlacklist.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXElementBlacklist.m; path = Classes/GTXElementBlacklist.m; sourceTree = SOURCE_ROOT; }; + 6133087C23FF2F53003F8D41 /* GTXReport.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXReport.m; path = Classes/GTXReport.m; sourceTree = SOURCE_ROOT; }; + 6133087D23FF2F53003F8D41 /* GTXSnapshotBuffer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXSnapshotBuffer.h; path = Classes/GTXSnapshotBuffer.h; sourceTree = SOURCE_ROOT; }; + 6133087E23FF2F53003F8D41 /* GTXElementBlacklist.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXElementBlacklist.h; path = Classes/GTXElementBlacklist.h; sourceTree = SOURCE_ROOT; }; + 6133087F23FF2F53003F8D41 /* GTXSnapshotContainer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXSnapshotContainer.h; path = Classes/GTXSnapshotContainer.h; sourceTree = SOURCE_ROOT; }; + 6133088023FF2F53003F8D41 /* GTXReport.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXReport.h; path = Classes/GTXReport.h; sourceTree = SOURCE_ROOT; }; 613C3F41204A09E8007D44A8 /* GTXiLib.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = GTXiLib.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 61A0C4DB2061896300DF0169 /* GTXiLibCore.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = GTXiLibCore.m; path = Classes/GTXiLibCore.m; sourceTree = SOURCE_ROOT; }; 61A0C4DC2061896300DF0169 /* GTXiLib.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = GTXiLib.h; path = Classes/GTXiLib.h; sourceTree = SOURCE_ROOT; }; @@ -289,6 +323,23 @@ 61ABAE8D204A0AF1006DBF0A /* Classes */ = { isa = PBXGroup; children = ( + 6133087E23FF2F53003F8D41 /* GTXElementBlacklist.h */, + 6133087B23FF2F53003F8D41 /* GTXElementBlacklist.m */, + 6133087723FF2F53003F8D41 /* GTXError.h */, + 6133087A23FF2F53003F8D41 /* GTXError.m */, + 6133088023FF2F53003F8D41 /* GTXReport.h */, + 6133087C23FF2F53003F8D41 /* GTXReport.m */, + 6133087D23FF2F53003F8D41 /* GTXSnapshotBuffer.h */, + 6133087923FF2F53003F8D41 /* GTXSnapshotBuffer.m */, + 6133087F23FF2F53003F8D41 /* GTXSnapshotContainer.h */, + 6133087823FF2F53003F8D41 /* GTXSnapshotContainer.m */, + 6133087623FF2F53003F8D41 /* UIColor+GTXAdditions.h */, + 6133087523FF2F53003F8D41 /* UIColor+GTXAdditions.m */, + 6133086E23FF2F39003F8D41 /* GTXArtifactCollector.h */, + 6133086B23FF2F39003F8D41 /* GTXArtifactCollector.m */, + 6133086F23FF2F39003F8D41 /* GTXArtifactImplementing.h */, + 6133086C23FF2F39003F8D41 /* GTXArtifactProcessor.h */, + 6133086D23FF2F39003F8D41 /* GTXArtifactProcessor.m */, 61A0C4DC2061896300DF0169 /* GTXiLib.h */, 61A0C4DD2061896300DF0169 /* GTXiLibCore.h */, 61A0C4DB2061896300DF0169 /* GTXiLibCore.m */, @@ -366,20 +417,28 @@ buildActionMask = 2147483647; files = ( 61284DDB211E437800F8FF29 /* GTXTestEnvironment.h in Headers */, + 6133087423FF2F39003F8D41 /* GTXArtifactImplementing.h in Headers */, 61ABAEB5204A0B0B006DBF0A /* GTXAccessibilityTree.h in Headers */, 61ABAECD204A0B0B006DBF0A /* GTXToolKit.h in Headers */, 61ABAEC8204A0B0B006DBF0A /* NSError+GTXAdditions.h in Headers */, + 6133088323FF2F53003F8D41 /* GTXError.h in Headers */, 61ABAEC4204A0B0B006DBF0A /* GTXImageAndColorUtils.h in Headers */, 61ABAEB6204A0B0B006DBF0A /* GTXCommon.h in Headers */, 61ABAEBD204A0B0B006DBF0A /* GTXAssertions.h in Headers */, 610CA3FD204DFC76008BAAA1 /* GTXPluginXCTestCase.h in Headers */, 61ABAECB204A0B0B006DBF0A /* GTXCheckBlock.h in Headers */, + 6133088A23FF2F53003F8D41 /* GTXElementBlacklist.h in Headers */, 61284DD6211E42CF00F8FF29 /* GTXBlacklistBlock.h in Headers */, + 6133088223FF2F53003F8D41 /* UIColor+GTXAdditions.h in Headers */, 61ABAEB9204A0B0B006DBF0A /* GTXChecking.h in Headers */, 61284E0A211E45C600F8FF29 /* GTXBlacklistFactory.h in Headers */, + 6133088C23FF2F53003F8D41 /* GTXReport.h in Headers */, + 6133088923FF2F53003F8D41 /* GTXSnapshotBuffer.h in Headers */, 61ABAEBE204A0B0B006DBF0A /* GTXImageRGBAData.h in Headers */, + 6133087123FF2F39003F8D41 /* GTXArtifactProcessor.h in Headers */, 61ABAEB2204A0B0B006DBF0A /* GTXErrorReporter.h in Headers */, 61ABAEC3204A0B0B006DBF0A /* GTXTestCase.h in Headers */, + 6133088B23FF2F53003F8D41 /* GTXSnapshotContainer.h in Headers */, 61ABAEB1204A0B0B006DBF0A /* GTXAnalytics.h in Headers */, 61284DD7211E42CF00F8FF29 /* GTXBlacklisting.h in Headers */, 61A0C4E02061896300DF0169 /* GTXiLibCore.h in Headers */, @@ -388,6 +447,7 @@ 61ABAEC1204A0B0B006DBF0A /* GTXTestSuite.h in Headers */, 61ABAEB7204A0B0B006DBF0A /* GTXLogging.h in Headers */, 61ABAEB0204A0B0B006DBF0A /* GTXAnalyticsUtils.h in Headers */, + 6133087323FF2F39003F8D41 /* GTXArtifactCollector.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -602,19 +662,27 @@ 61ABAECC204A0B0B006DBF0A /* GTXErrorReporter.m in Sources */, 61ABAEB4204A0B0B006DBF0A /* GTXTestSuite.m in Sources */, 61ABAEB3204A0B0B006DBF0A /* GTXImageAndColorUtils.m in Sources */, + 6133087223FF2F39003F8D41 /* GTXArtifactProcessor.m in Sources */, 61284E0B211E45C600F8FF29 /* GTXBlacklistFactory.m in Sources */, 61ABAEC2204A0B0B006DBF0A /* GTXToolKit.m in Sources */, + 6133088123FF2F53003F8D41 /* UIColor+GTXAdditions.m in Sources */, 61ABAEC9204A0B0B006DBF0A /* GTXAnalytics.m in Sources */, 610CA3FE204DFC76008BAAA1 /* GTXPluginXCTestCase.m in Sources */, 61284DDA211E437800F8FF29 /* GTXTestEnvironment.m in Sources */, 61ABAEB8204A0B0B006DBF0A /* GTXAnalyticsUtils.m in Sources */, + 6133088623FF2F53003F8D41 /* GTXError.m in Sources */, 61284DD5211E42CF00F8FF29 /* GTXBlacklistBlock.m in Sources */, + 6133088723FF2F53003F8D41 /* GTXElementBlacklist.m in Sources */, 61ABAEBF204A0B0B006DBF0A /* NSError+GTXAdditions.m in Sources */, 61ABAEBC204A0B0B006DBF0A /* GTXAccessibilityTree.m in Sources */, + 6133088523FF2F53003F8D41 /* GTXSnapshotBuffer.m in Sources */, + 6133088423FF2F53003F8D41 /* GTXSnapshotContainer.m in Sources */, 61ABAEC0204A0B0B006DBF0A /* GTXCheckBlock.m in Sources */, 61ABAEBB204A0B0B006DBF0A /* GTXTestCase.m in Sources */, + 6133087023FF2F39003F8D41 /* GTXArtifactCollector.m in Sources */, 61ABAECE204A0B0B006DBF0A /* GTXImageRGBAData.m in Sources */, 61A0C4DE2061896300DF0169 /* GTXiLibCore.m in Sources */, + 6133088823FF2F53003F8D41 /* GTXReport.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Tests/UnitTests/GTXAnalyticsTests.m b/Tests/UnitTests/GTXAnalyticsTests.m index 748cabf..ea0e4c7 100644 --- a/Tests/UnitTests/GTXAnalyticsTests.m +++ b/Tests/UnitTests/GTXAnalyticsTests.m @@ -56,11 +56,11 @@ - (void)testCheckElementReportsAnalyticsCorrectly { }]; NSObject *failingElement = [self newAccessibleElement]; NSObject *passingElement = [self newAccessibleElement]; - id check = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element == passingElement; - }]; + id check = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element == passingElement; + }]; [toolkit registerCheck:check]; NSError *error; XCTAssertEqual(successEventsCount, 0); @@ -94,35 +94,88 @@ - (void)testCheckElementsFromRootElementsReportsAnalyticsCorrectly { failureEventsCount += 1; } }]; - NSObject *root = [self newInAccessibleElement]; + NSObject *root = [self newInaccessibleElement]; NSObject *child1 = [self newAccessibleElement]; - NSObject *child2 = [self newInAccessibleElement]; - id checkFailIfChild1 = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element != child1; - }]; - [self createTreeFromPreOrderTraversal:@[root, - child1, child2, [NSNull null], - ]]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; [toolkit registerCheck:checkFailIfChild1]; NSError *error; XCTAssertEqual(successEventsCount, 0); XCTAssertEqual(failureEventsCount, 0); - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:nil]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:nil]); XCTAssertEqual(successEventsCount, 0); XCTAssertEqual(failureEventsCount, 1); - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:&error]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:&error]); XCTAssertEqual(successEventsCount, 0); XCTAssertEqual(failureEventsCount, 2); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:nil]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:nil]); XCTAssertEqual(successEventsCount, 1); XCTAssertEqual(failureEventsCount, 2); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:&error]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:&error]); + XCTAssertEqual(successEventsCount, 2); + XCTAssertEqual(failureEventsCount, 2); +} + +- (void)testResultFromCheckingAllElementsFromRootElementsReportsAnalyticsCorrectly { + GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; + __block NSInteger successEventsCount = 0; + __block NSInteger failureEventsCount = 0; + [GTXAnalytics setHandler:^(GTXAnalyticsEvent event) { + if (event == GTXAnalyticsEventChecksPerformed) { + successEventsCount += 1; + } else { + failureEventsCount += 1; + } + }]; + NSObject *root = [self newInaccessibleElement]; + NSObject *child1 = [self newAccessibleElement]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; + [toolkit registerCheck:checkFailIfChild1]; + XCTAssertEqual(successEventsCount, 0); + XCTAssertEqual(failureEventsCount, 0); + + XCTAssertFalse( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ root ]] allChecksPassed]); + XCTAssertEqual(successEventsCount, 0); + XCTAssertEqual(failureEventsCount, 1); + + XCTAssertFalse( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ root ]] allChecksPassed]); + XCTAssertEqual(successEventsCount, 0); + XCTAssertEqual(failureEventsCount, 2); + + XCTAssertTrue( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ child2 ]] allChecksPassed]); + XCTAssertEqual(successEventsCount, 1); + XCTAssertEqual(failureEventsCount, 2); + + XCTAssertTrue( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ child2 ]] allChecksPassed]); XCTAssertEqual(successEventsCount, 2); XCTAssertEqual(failureEventsCount, 2); } @@ -141,17 +194,20 @@ - (void)testAnalyticsCanBeDisabled { GTXAnalytics.enabled = NO; - NSObject *root = [self newInAccessibleElement]; + NSObject *root = [self newInaccessibleElement]; NSObject *child1 = [self newAccessibleElement]; - NSObject *child2 = [self newInAccessibleElement]; - id checkFailIfChild1 = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element != child1; - }]; - [self createTreeFromPreOrderTraversal:@[root, - child1, child2, [NSNull null], - ]]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; [toolkit registerCheck:checkFailIfChild1]; NSError *error; XCTAssertEqual(successEventsCount, 0); @@ -161,10 +217,14 @@ - (void)testAnalyticsCanBeDisabled { XCTAssertTrue([toolkit checkElement:root error:&error]); XCTAssertFalse([toolkit checkElement:child1 error:nil]); XCTAssertFalse([toolkit checkElement:child1 error:&error]); - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:nil]); - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:&error]); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:nil]); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:&error]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:nil]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:&error]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:nil]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:&error]); + XCTAssertFalse( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ root ]] allChecksPassed]); + XCTAssertTrue( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ child2 ]] allChecksPassed]); XCTAssertEqual(successEventsCount, 0); XCTAssertEqual(failureEventsCount, 0); diff --git a/Tests/UnitTests/GTXBaseTestCase.h b/Tests/UnitTests/GTXBaseTestCase.h index 74fd27c..0b4feb3 100644 --- a/Tests/UnitTests/GTXBaseTestCase.h +++ b/Tests/UnitTests/GTXBaseTestCase.h @@ -24,8 +24,7 @@ extern GTXCheckHandlerBlock noOpCheckBlock; @interface GTXBaseTestCase : XCTestCase - (NSObject *)newAccessibleElement; -- (NSObject *)newInAccessibleElement; +- (NSObject *)newInaccessibleElement; - (void)createTreeFromPreOrderTraversal:(NSArray *)preOrderTraversal; @end - diff --git a/Tests/UnitTests/GTXBaseTestCase.m b/Tests/UnitTests/GTXBaseTestCase.m index 79b0a44..fa573ea 100644 --- a/Tests/UnitTests/GTXBaseTestCase.m +++ b/Tests/UnitTests/GTXBaseTestCase.m @@ -31,7 +31,7 @@ - (NSObject *)newAccessibleElement { return element; } -- (NSObject *)newInAccessibleElement { +- (NSObject *)newInaccessibleElement { NSObject *element = [[NSObject alloc] init]; element.isAccessibilityElement = NO; return element; diff --git a/Tests/UnitTests/GTXResultTests.m b/Tests/UnitTests/GTXResultTests.m new file mode 100644 index 0000000..6b4a7ee --- /dev/null +++ b/Tests/UnitTests/GTXResultTests.m @@ -0,0 +1,65 @@ +// +// Copyright 2018 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#import +#import + +#import "GTXResult.h" +#import "NSError+GTXAdditions.h" + +static NSString *const kGTXTestDomain = @"kGTXTESTDomain"; +static const NSInteger kGTXTestNonZeroElements = 9; + +@interface GTXResultTests : XCTestCase +@end + +@implementation GTXResultTests + +- (void)testResultReportsSuccessForZeroErrorsZeroElementsScanned { + XCTAssertTrue([[[GTXResult alloc] initWithErrorsFound:@[] elementsScanned:0] allChecksPassed]); +} + +- (void)testResultReportsSuccessForZeroErrorsNonZeroElementsScanned { + XCTAssertTrue([[[GTXResult alloc] initWithErrorsFound:@[] + elementsScanned:kGTXTestNonZeroElements] allChecksPassed]); +} + +- (void)testResultResultsInNilAggregateForZeroErrors { + XCTAssertNil([[[GTXResult alloc] initWithErrorsFound:@[] elementsScanned:0] aggregatedError]); + XCTAssertNil([[[GTXResult alloc] initWithErrorsFound:@[] + elementsScanned:kGTXTestNonZeroElements] aggregatedError]); +} + +- (void)testAggregateErrorContainsIndividualIssuesWithOneIssue { + NSError *error = [NSError errorWithDomain:kGTXTestDomain code:0 userInfo:nil]; + GTXResult *result = [[GTXResult alloc] initWithErrorsFound:@[ error ] + elementsScanned:kGTXTestNonZeroElements]; + XCTAssertEqualObjects([[result aggregatedError] userInfo][kGTXErrorUnderlyingErrorsKey], + @[ error ]); +} + +- (void)testAggregateErrorContainsIndividualIssuesWithMultipleIssues { + NSError *error1 = [NSError errorWithDomain:kGTXTestDomain code:0 userInfo:nil]; + NSError *error2 = [NSError errorWithDomain:kGTXTestDomain code:0 userInfo:nil]; + NSSet *actualErrorsSet = [NSSet setWithObjects:error1, error2, nil]; + GTXResult *result = [[GTXResult alloc] initWithErrorsFound:[actualErrorsSet allObjects] + elementsScanned:kGTXTestNonZeroElements]; + XCTAssertEqualObjects( + actualErrorsSet, + [NSSet setWithArray:[[result aggregatedError] userInfo][kGTXErrorUnderlyingErrorsKey]]); +} + +@end diff --git a/Tests/UnitTests/GTXToolKitTests.m b/Tests/UnitTests/GTXToolKitTests.m index bb092db..5305842 100644 --- a/Tests/UnitTests/GTXToolKitTests.m +++ b/Tests/UnitTests/GTXToolKitTests.m @@ -17,9 +17,9 @@ #import #import -#import "GTXToolKit.h" #import "GTXAnalytics.h" #import "GTXBlacklistFactory.h" +#import "GTXToolKit.h" #import "GTXBaseTestCase.h" @interface GTXTestElementClass1 : UIAccessibilityElement @@ -63,11 +63,11 @@ - (void)testCheckElementReportsFailures { GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; NSObject *failingElement = [self newAccessibleElement]; NSObject *passingElement = [self newAccessibleElement]; - id check = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element == passingElement; - }]; + id check = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element == passingElement; + }]; [toolkit registerCheck:check]; NSError *error; XCTAssertTrue([toolkit checkElement:passingElement error:nil]); @@ -78,23 +78,49 @@ - (void)testCheckElementReportsFailures { - (void)testCheckElementsFromRootElementsReportsFailures { GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; - NSObject *root = [self newInAccessibleElement]; + NSObject *root = [self newInaccessibleElement]; NSObject *child1 = [self newAccessibleElement]; - NSObject *child2 = [self newInAccessibleElement]; - id checkFailIfChild1 = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element != child1; - }]; - [self createTreeFromPreOrderTraversal:@[root, - child1, child2, [NSNull null], - ]]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; [toolkit registerCheck:checkFailIfChild1]; NSError *error; - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:nil]); - XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[root] error:&error]); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:nil]); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[child2] error:&error]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:nil]); + XCTAssertFalse([toolkit checkAllElementsFromRootElements:@[ root ] error:&error]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:nil]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ child2 ] error:&error]); +} + +- (void)testResultFromCheckingAllElementsFromRootElementsReportsFailures { + GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; + NSObject *root = [self newInaccessibleElement]; + NSObject *child1 = [self newAccessibleElement]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; + [toolkit registerCheck:checkFailIfChild1]; + XCTAssertFalse( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ root ]] allChecksPassed]); + XCTAssertTrue( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ child2 ]] allChecksPassed]); } - (void)testCheckElementsFromRootElementsSkipsHiddenAXElements { @@ -102,33 +128,58 @@ - (void)testCheckElementsFromRootElementsSkipsHiddenAXElements { NSObject *root = [self newAccessibleElement]; // Since root is an accessibile element its children are hidden. NSObject *child1 = [self newAccessibleElement]; - NSObject *child2 = [self newInAccessibleElement]; - id checkFailIfChild1 = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element != child1; - }]; - [self createTreeFromPreOrderTraversal:@[root, - child1, child2, [NSNull null], - ]]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; [toolkit registerCheck:checkFailIfChild1]; NSError *error; - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[root] error:nil]); - XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[root] error:&error]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ root ] error:nil]); + XCTAssertTrue([toolkit checkAllElementsFromRootElements:@[ root ] error:&error]); +} + +- (void)testResultFromCheckingAllElementsFromRootElementsSkipsHiddenAXElements { + GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; + NSObject *root = [self newAccessibleElement]; + // Since root is an accessibile element its children are hidden. + NSObject *child1 = [self newAccessibleElement]; + NSObject *child2 = [self newInaccessibleElement]; + id checkFailIfChild1 = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != child1; + }]; + [self createTreeFromPreOrderTraversal:@[ + root, + child1, + child2, + [NSNull null], + ]]; + [toolkit registerCheck:checkFailIfChild1]; + XCTAssertTrue( + [[toolkit resultFromCheckingAllElementsFromRootElements:@[ root ]] allChecksPassed]); } - (void)testBlacklistAPICanSkipElementsFromChecks { GTXToolKit *toolkit = [GTXToolKit toolkitWithNoChecks]; NSObject *failingElement = [self newAccessibleElement]; - id check = [GTXToolKit checkWithName:@"Foo" - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return NO; - }]; + id check = + [GTXToolKit checkWithName:@"Foo" + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return NO; + }]; [toolkit registerCheck:check]; XCTAssertFalse([toolkit checkElement:failingElement error:nil]); - [toolkit registerBlacklist: - [GTXBlacklistFactory blacklistWithClassName:NSStringFromClass([failingElement class])]]; + [toolkit registerBlacklist:[GTXBlacklistFactory + blacklistWithClassName:NSStringFromClass([failingElement class])]]; XCTAssertTrue([toolkit checkElement:failingElement error:nil]); } @@ -146,16 +197,16 @@ - (void)testBlacklistAPICanSkipElementsFromSpecificChecks { NSString *check1Name = @"Check 1"; NSString *check2Name = @"Check 2"; - id check1 = [GTXToolKit checkWithName:check1Name - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return NO; - }]; - id check2 = [GTXToolKit checkWithName:check2Name - block:^BOOL(id _Nonnull element, - GTXErrorRefType errorOrNil) { - return element != allChecksFailingElement; - }]; + id check1 = + [GTXToolKit checkWithName:check1Name + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return NO; + }]; + id check2 = + [GTXToolKit checkWithName:check2Name + block:^BOOL(id _Nonnull element, GTXErrorRefType errorOrNil) { + return element != allChecksFailingElement; + }]; GTXToolKit *toolkit1 = [GTXToolKit toolkitWithNoChecks]; [toolkit1 registerCheck:check1]; @@ -164,24 +215,26 @@ - (void)testBlacklistAPICanSkipElementsFromSpecificChecks { XCTAssertFalse([toolkit1 checkElement:allChecksFailingElement error:nil]); XCTAssertFalse([toolkit1 checkElement:check3FailingElement error:nil]); - [toolkit1 registerBlacklist: - [GTXBlacklistFactory blacklistWithAccessibilityIdentifier:@"check3FailingElement" - checkName:check1Name]]; + [toolkit1 registerBlacklist:[GTXBlacklistFactory + blacklistWithAccessibilityIdentifier:@"check3FailingElement" + checkName:check1Name]]; XCTAssertFalse([toolkit1 checkElement:check1FailingElement error:nil]); XCTAssertFalse([toolkit1 checkElement:allChecksFailingElement error:nil]); XCTAssertTrue([toolkit1 checkElement:check3FailingElement error:nil]); - [toolkit1 registerBlacklist: - [GTXBlacklistFactory blacklistWithClassName:NSStringFromClass([check1FailingElement class])]]; - [toolkit1 registerBlacklist: - [GTXBlacklistFactory blacklistWithClassName:NSStringFromClass([check1FailingElement class])]]; + [toolkit1 registerBlacklist:[GTXBlacklistFactory + blacklistWithClassName:NSStringFromClass( + [check1FailingElement class])]]; + [toolkit1 registerBlacklist:[GTXBlacklistFactory + blacklistWithClassName:NSStringFromClass( + [check1FailingElement class])]]; XCTAssertTrue([toolkit1 checkElement:check1FailingElement error:nil]); XCTAssertFalse([toolkit1 checkElement:allChecksFailingElement error:nil]); XCTAssertTrue([toolkit1 checkElement:check3FailingElement error:nil]); NSString *allChecksFailingElementClass = NSStringFromClass([allChecksFailingElement class]); - [toolkit1 registerBlacklist: - [GTXBlacklistFactory blacklistWithClassName:allChecksFailingElementClass]]; + [toolkit1 + registerBlacklist:[GTXBlacklistFactory blacklistWithClassName:allChecksFailingElementClass]]; XCTAssertTrue([toolkit1 checkElement:allChecksFailingElement error:nil]); }