[AI-FSSDK] [FSSDK-12735] Add holdout exclusion logic for Targeted Delivery rules (#1171)

Implements holdout exclusion logic for Targeted Delivery rules. When a holdout's `excludeTargetedDeliveries` flag is true, users bucketed into that holdout will still be served Targeted Delivery (rollout) rules normally, while A/B Test and Multi-Armed Bandit experiment rules continue to be blocked by the holdout.

## Changes

- Added `excludeTargetedDeliveries` field to the Holdout interface in shared types
- Updated `resolveVariationForFlag` to evaluate delivery rules when holdout has exclusion enabled, falling back to holdout decision if no delivery rule matches
- Added decision reason logging for when Targeted Delivery rules are excluded from holdout logic

---------

Co-authored-by: Raju Ahmed <raju.ahmed@optimizely.com>

Author: esrakartalOpt

lib/core/decision_service/index.spec.ts

@@ -19,7 +19,7 @@ import {
   USER_HAS_NO_FORCED_VARIATION
 } from 'log_message';
 import { beforeEach, describe, expect, it, MockInstance, vi } from 'vitest';
-import { CMAB_DUMMY_ENTITY_ID, CMAB_FETCH_FAILED, DecisionService } from '.';
+import { CMAB_DUMMY_ENTITY_ID, CMAB_FETCH_FAILED, DecisionService, TARGETED_DELIVERY_EXCLUDED_FROM_HOLDOUT } from '.';
 import OptimizelyUserContext from '../../optimizely_user_context';
 import { createProjectConfig, ProjectConfig } from '../../project_config/project_config';
 import { BucketerParams, Experiment, ExperimentBucketMap, Holdout, OptimizelyDecideOption, UserAttributes, UserProfile } from '../../shared_types';
@@ -2196,6 +2196,170 @@ describe('DecisionService', () => {
           decisionSource: DECISION_SOURCES.HOLDOUT,
         });
       });
+
+      describe('excludeTargetedDeliveries', () => {
+        const getExcludeTDDatafile = (excludeTargetedDeliveries?: boolean) => {
+          const datafile = getHoldoutTestDatafile();
+          datafile.holdouts = datafile.holdouts.map((holdout: any) => {
+            if (holdout.id === 'holdout_running_id') {
+              return {
+                ...holdout,
+                ...(excludeTargetedDeliveries !== undefined ? { excludeTargetedDeliveries } : {}),
+              };
+            }
+            return holdout;
+          });
+          return datafile;
+        };
+
+        it('should return holdout variation immediately when excludeTargetedDeliveries is false', async () => {
+          const { decisionService } = getDecisionService();
+          const config = createProjectConfig(JSON.stringify(getExcludeTDDatafile(false)));
+          const user = new OptimizelyUserContext({
+            optimizely: {} as any,
+            userId: 'tester',
+            attributes: { age: 20 },
+          });
+
+          const feature = config.featureKeyMap['flag_1'];
+          const value = decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+          const variation = (await value)[0];
+
+          expect(variation.result).toEqual({
+            experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
+            variation: config.variationIdMap['holdout_variation_running_id'],
+            decisionSource: DECISION_SOURCES.HOLDOUT,
+          });
+        });
+
+        it('should return holdout variation immediately when excludeTargetedDeliveries is undefined', async () => {
+          const { decisionService } = getDecisionService();
+          const config = createProjectConfig(JSON.stringify(getExcludeTDDatafile(undefined)));
+          const user = new OptimizelyUserContext({
+            optimizely: {} as any,
+            userId: 'tester',
+            attributes: { age: 20 },
+          });
+
+          const feature = config.featureKeyMap['flag_1'];
+          const value = decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+          const variation = (await value)[0];
+
+          expect(variation.result).toEqual({
+            experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
+            variation: config.variationIdMap['holdout_variation_running_id'],
+            decisionSource: DECISION_SOURCES.HOLDOUT,
+          });
+        });
+
+        it('should return delivery variation with holdout info when excludeTargetedDeliveries is true and delivery matches', async () => {
+          const { decisionService } = getDecisionService();
+          const datafile = getExcludeTDDatafile(true);
+
+          const config = createProjectConfig(JSON.stringify(datafile));
+          const user = new OptimizelyUserContext({
+            optimizely: {} as any,
+            userId: 'tester',
+            attributes: { age: 20 },
+          });
+
+          mockBucket.mockImplementation((param: BucketerParams) => {
+            if (param.experimentKey === 'delivery_1') {
+              return { result: '5004', reasons: [] };
+            }
+            if (param.experimentKey === 'holdout_running') {
+              return { result: 'holdout_variation_running_id', reasons: [] };
+            }
+            return { result: null, reasons: [] };
+          });
+
+          const feature = config.featureKeyMap['flag_1'];
+          const value = decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+          const variation = (await value)[0];
+
+          expect(variation.result).toEqual({
+            experiment: config.experimentKeyMap['delivery_1'],
+            variation: config.variationIdMap['5004'],
+            decisionSource: DECISION_SOURCES.ROLLOUT,
+            holdout: {
+              experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
+              variation: config.variationIdMap['holdout_variation_running_id'],
+            },
+          });
+
+          const reasonStrings = variation.reasons.map((r: any) => typeof r === 'string' ? r : r[0]);
+          expect(reasonStrings).toContain(TARGETED_DELIVERY_EXCLUDED_FROM_HOLDOUT);
+        });
+
+        it('should return null decision with holdout info when excludeTargetedDeliveries is true but no delivery matches', async () => {
+          const { decisionService } = getDecisionService();
+          const datafile = getExcludeTDDatafile(true);
+
+          const config = createProjectConfig(JSON.stringify(datafile));
+          const user = new OptimizelyUserContext({
+            optimizely: {} as any,
+            userId: 'tester',
+            attributes: { age: 20 },
+          });
+
+          mockBucket.mockImplementation((param: BucketerParams) => {
+            if (param.experimentKey === 'holdout_running') {
+              return { result: 'holdout_variation_running_id', reasons: [] };
+            }
+            return { result: null, reasons: [] };
+          });
+
+          const feature = config.featureKeyMap['flag_1'];
+          const value = decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+          const variation = (await value)[0];
+
+          expect(variation.result).toEqual({
+            experiment: null,
+            variation: null,
+            decisionSource: DECISION_SOURCES.ROLLOUT,
+            holdout: {
+              experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
+              variation: config.variationIdMap['holdout_variation_running_id'],
+            },
+          });
+        });
+
+        it('should still block experiment rules when excludeTargetedDeliveries is true', async () => {
+          const { decisionService } = getDecisionService();
+          const datafile = getExcludeTDDatafile(true);
+
+          const config = createProjectConfig(JSON.stringify(datafile));
+          const user = new OptimizelyUserContext({
+            optimizely: {} as any,
+            userId: 'tester',
+            attributes: { age: 20 },
+          });
+
+          mockBucket.mockImplementation((param: BucketerParams) => {
+            if (param.experimentKey === 'holdout_running') {
+              return { result: 'holdout_variation_running_id', reasons: [] };
+            }
+            if (param.experimentKey === 'exp_1') {
+              return { result: '5001', reasons: [] };
+            }
+            return { result: null, reasons: [] };
+          });
+
+          const feature = config.featureKeyMap['flag_1'];
+          const value = decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+          const variation = (await value)[0];
+
+          expect(variation.result).toEqual({
+            experiment: null,
+            variation: null,
+            decisionSource: DECISION_SOURCES.ROLLOUT,
+            holdout: {
+              experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
+              variation: config.variationIdMap['holdout_variation_running_id'],
+            },
+          });
+        });
+      });
     });
   });
 
@@ -3101,6 +3265,34 @@ describe('DecisionService', () => {
       expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.FEATURE_TEST);
       expect(value[0].result.variation?.key).toBe('variation_1');
     });
+
+    it('local holdout ignores excludeTargetedDeliveries and applies normally', async () => {
+      const datafile = makeLocalHoldoutDatafile('2001');
+      (datafile as any).localHoldouts[0].excludeTargetedDeliveries = true;
+      const config = createProjectConfig(JSON.stringify(datafile));
+      const { decisionService } = getDecisionService();
+
+      mockBucket.mockImplementation((params: BucketerParams) => {
+        if (params.experimentId === 'local_holdout_id') {
+          return { result: 'local_holdout_variation_id', reasons: [] };
+        }
+        return { result: null, reasons: [] };
+      });
+
+      const user = new OptimizelyUserContext({
+        optimizely: {} as any,
+        userId: 'user1',
+        attributes: { age: 15 },
+      });
+
+      const feature = config.featureKeyMap['flag_1'];
+      const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
+
+      // Local holdout applies normally even with excludeTargetedDeliveries set
+      expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.HOLDOUT);
+      expect(value[0].result.experiment?.id).toBe('local_holdout_id');
+      expect(value[0].result.variation?.id).toBe('local_holdout_variation_id');
+    });
   });
 });
 

lib/core/decision_service/index.ts

@@ -119,12 +119,17 @@ export const USER_MEETS_CONDITIONS_FOR_HOLDOUT = 'User %s meets conditions for h
 export const USER_DOESNT_MEET_CONDITIONS_FOR_HOLDOUT = 'User %s does not meet conditions for holdout %s.';
 export const USER_BUCKETED_INTO_HOLDOUT_VARIATION = 'User %s is in variation %s of holdout %s.';
 export const USER_NOT_BUCKETED_INTO_HOLDOUT_VARIATION = 'User %s is in no holdout variation.';
+export const TARGETED_DELIVERY_EXCLUDED_FROM_HOLDOUT = 'Holdout "%s" has excludeTargetedDeliveries enabled, continuing to rollout evaluation.';
 
 export interface DecisionObj {
   experiment: Experiment | Holdout | null;
   variation: Variation | null;
   decisionSource: DecisionSource;
   cmabUuid?: string;
+  holdout?: {
+    experiment: Holdout;
+    variation: Variation;
+  };
 }
 
 interface DecisionServiceOptions {
@@ -913,11 +918,7 @@ export class DecisionService {
     options: DecideOptionsMap): Value<OP, DecisionResult[]> {
     const userId = user.getUserId();
     const attributes = user.getAttributes();
-    const decisions: DecisionResponse<DecisionObj>[] = [];
-    // const userProfileTracker : UserProfileTracker = {
-    //   isProfileUpdated: false,
-    //   userProfile: null,
-    // }
+
     const shouldIgnoreUPS = !!options[OptimizelyDecideOption.IGNORE_USER_PROFILE_SERVICE];
 
     const userProfileTrackerValue: Value<OP, Maybe<UserProfileTracker>> = shouldIgnoreUPS ? Value.of(op, undefined)
@@ -969,19 +970,41 @@ export class DecisionService {
     // getGlobalHoldouts() returns holdouts with includedRules == null/undefined.
     const globalHoldouts = getGlobalHoldouts(configObj);
 
+    let appliedHoldout : DecisionObj['holdout'] | undefined = undefined;
+
     for (const holdout of globalHoldouts) {
       const holdoutDecision = this.getVariationForHoldout(configObj, holdout, user);
       decideReasons.push(...holdoutDecision.reasons);
 
       if (holdoutDecision.result.variation) {
+        if (holdout.excludeTargetedDeliveries) {
+          this.logger?.info(TARGETED_DELIVERY_EXCLUDED_FROM_HOLDOUT, holdout.key);
+          decideReasons.push([TARGETED_DELIVERY_EXCLUDED_FROM_HOLDOUT, holdout.key]);
+          appliedHoldout = {
+            experiment: holdout,
+            variation: holdoutDecision.result.variation
+          }
+          break;
+        }
+
         return Value.of(op, {
           result: holdoutDecision.result,
           reasons: decideReasons,
         });
       }
     }
 
-    return this.getVariationForFeatureExperiment(op, configObj, feature, user, decideOptions, userProfileTracker).then((experimentDecision) => {
+    const experimentDecision: Value<OP, DecisionResult> = appliedHoldout ? 
+      Value.of(op, {
+        reasons: decideReasons,
+        result: {
+          experiment: null,
+          variation: null,
+          decisionSource: DECISION_SOURCES.FEATURE_TEST,
+        }
+      }) : this.getVariationForFeatureExperiment(op, configObj, feature, user, decideOptions, userProfileTracker);
+
+    return experimentDecision.then((experimentDecision) => {
       if (experimentDecision.error || experimentDecision.result.variation !== null) {
         return Value.of(op, {
           ...experimentDecision,
@@ -990,20 +1013,24 @@ export class DecisionService {
       }
 
       decideReasons.push(...experimentDecision.reasons);
-      
+
       const rolloutDecision = this.getVariationForRollout(configObj, feature, user);
       decideReasons.push(...rolloutDecision.reasons);
       const rolloutDecisionResult = rolloutDecision.result;
       const userId = user.getUserId();
-  
+
       if (rolloutDecisionResult.variation) {
         this.logger?.debug(USER_IN_ROLLOUT, userId, feature.key);
         decideReasons.push([USER_IN_ROLLOUT, userId, feature.key]);
       } else {
         this.logger?.debug(USER_NOT_IN_ROLLOUT, userId, feature.key);
         decideReasons.push([USER_NOT_IN_ROLLOUT, userId, feature.key]);
       }
-  
+
+      if (appliedHoldout) {
+        rolloutDecisionResult.holdout = appliedHoldout;
+      }
+
       return Value.of(op, {
         result: rolloutDecisionResult,
         reasons: decideReasons,

lib/optimizely/index.spec.ts

@@ -872,6 +872,66 @@ describe('Optimizely', () => {
         }),
       });
     });
+
+    it('should dispatch separate holdout impression when decisionObj.holdout is populated', async () => {
+      const processSpy = vi.spyOn(eventProcessor, 'process');
+      const notificationSpyLocal = vi.fn();
+      optimizely.notificationCenter.addNotificationListener(
+        NOTIFICATION_TYPES.DECISION,
+        notificationSpyLocal
+      );
+
+      const holdoutExperiment = projectConfig.holdouts[0];
+      const holdoutVariation = projectConfig.holdouts[0].variations[0];
+      const rolloutExperiment = projectConfig.experimentKeyMap['delivery_1'] || Object.values(projectConfig.experimentIdMap)[0];
+      const rolloutVariation = rolloutExperiment?.variations?.[0] || { id: 'test_var_id', key: 'test_var_key', variables: [] };
+
+      vi.spyOn(decisionService, 'resolveVariationsForFeatureList').mockImplementation(() => {
+        return Value.of('async', [{
+          error: false,
+          result: {
+            variation: rolloutVariation,
+            experiment: rolloutExperiment,
+            decisionSource: DECISION_SOURCES.ROLLOUT,
+            holdout: {
+              experiment: holdoutExperiment,
+              variation: holdoutVariation,
+            },
+          },
+          reasons: [],
+        }]);
+      });
+
+      const user = new OptimizelyUserContext({
+        optimizely,
+        userId: 'test_user',
+        attributes: {},
+      });
+
+      await optimizely.decideAsync(user, 'flag_1', []);
+
+      // The holdout impression is dispatched even when the main rollout impression is not
+      // (sendFlagDecisions not set). Verify at least one impression is a holdout event.
+      expect(processSpy).toHaveBeenCalled();
+
+      const holdoutEvent = processSpy.mock.calls.find(
+        (call: any) => (call[0] as ImpressionEvent).ruleType === 'holdout'
+      );
+      expect(holdoutEvent).toBeDefined();
+      const holdoutImpression = holdoutEvent![0] as ImpressionEvent;
+      expect(holdoutImpression.ruleKey).toBe('holdout_test_key');
+      expect(holdoutImpression.ruleType).toBe('holdout');
+      expect(holdoutImpression.enabled).toBe(false);
+
+      expect(notificationSpyLocal).toHaveBeenCalledWith(
+        expect.objectContaining({
+          type: DECISION_NOTIFICATION_TYPES.FLAG,
+          decisionInfo: expect.objectContaining({
+            decisionEventDispatched: true,
+          }),
+        })
+      );
+    });
   });
 
   it('should flush eventProcessor and odpManager on flushImmediately()', async () => {