Add feature gates and experiment exposure telemetry events (#221)

Author: marcomura

src/analytics.ts

@@ -91,13 +91,27 @@ export async function featureFlagClientInitializedEvent(success: true): Promise<
 export async function featureFlagClientInitializedEvent(
     success: false,
     errorType: ClientInitializedErrorType,
+    reason: string,
 ): Promise<TrackEvent>;
 export async function featureFlagClientInitializedEvent(
     success: boolean,
     errorType?: ClientInitializedErrorType,
+    reason?: string,
 ): Promise<TrackEvent> {
     return trackEvent('initialized', 'featureFlagClient', {
-        attributes: { success, errorType: errorType ?? 0 },
+        attributes: { success, errorType: errorType ?? 0, reason },
+    });
+}
+
+// debugging event, meant to measure the exposure rate of a feature flag or an experiment
+export async function featureGateExposureBoolEvent(
+    ffName: string,
+    success: boolean,
+    value: boolean,
+    errorType: number,
+): Promise<TrackEvent> {
+    return trackEvent('gateExposureBool', 'featureFlagClient', {
+        attributes: { ffName, success, value, errorType },
     });
 }
 

src/container.ts

@@ -207,25 +207,23 @@ export class Container {
         } catch (err) {
             const error = err as FeatureFlagClientInitError;
             Logger.debug(`FeatureFlagClient: Failed to initialize the client: ${error.reason}`);
-            featureFlagClientInitializedEvent(false, error.errorType).then((e) => {
+            featureFlagClientInitializedEvent(false, error.errorType, error.reason).then((e) => {
                 this.analyticsClient.sendTrackEvent(e);
             });
         }
 
+        FeatureFlagClient.checkExperimentBooleanValueWithInstrumentation(Experiments.AtlascodeAA);
+        FeatureFlagClient.checkGateValueWithInstrumentation(Features.NoOpFeature);
+
         this.initializeUriHandler(context, this._analyticsApi, this._bitbucketHelper);
         this.initializeNewSidebarView(context, config);
-        this.initializeAAExperiment();
     }
 
     private static getAnalyticsEnable(): boolean {
         const telemetryConfig = workspace.getConfiguration('telemetry');
         return telemetryConfig.get<boolean>('enableTelemetry', true);
     }
 
-    private static initializeAAExperiment() {
-        FeatureFlagClient.checkExperimentValue(Experiments.AtlascodeAA);
-    }
-
     private static initializeUriHandler(
         context: ExtensionContext,
         analyticsApi: VSCAnalyticsApi,

src/util/featureFlags/client.ts

@@ -2,7 +2,7 @@ import FeatureGates, { FeatureGateEnvironment, Identifiers } from '@atlaskit/fea
 import { AnalyticsClient } from '../../analytics-node-client/src/client.min';
 import { AnalyticsClientMapper } from './analytics';
 import { ExperimentGates, ExperimentGateValues, Experiments, FeatureGateValues, Features } from './features';
-import { ClientInitializedErrorType } from '../../analytics';
+import { ClientInitializedErrorType, featureGateExposureBoolEvent } from '../../analytics';
 import { Logger } from '../../logger';
 
 export type FeatureFlagClientOptions = {
@@ -18,7 +18,8 @@ export class FeatureFlagClientInitError {
 }
 
 export abstract class FeatureFlagClient {
-    private static analyticsClient: AnalyticsClientMapper;
+    private static analyticsClient: AnalyticsClient;
+    private static analyticsClientMapper: AnalyticsClientMapper;
 
     private static featureGateOverrides: FeatureGateValues;
     private static experimentValueOverride: ExperimentGateValues;
@@ -44,7 +45,9 @@ export abstract class FeatureFlagClient {
         }
 
         Logger.debug(`FeatureGates: initializing, target: ${targetApp}, environment: ${environment}`);
-        this.analyticsClient = new AnalyticsClientMapper(options.analyticsClient, options.identifiers);
+
+        this.analyticsClient = options.analyticsClient;
+        this.analyticsClientMapper = new AnalyticsClientMapper(options.analyticsClient, options.identifiers);
 
         try {
             await FeatureGates.initialize(
@@ -53,7 +56,7 @@ export abstract class FeatureFlagClient {
                     environment,
                     targetApp,
                     fetchTimeoutMs: Number.parseInt(timeout),
-                    analyticsWebClient: Promise.resolve(this.analyticsClient),
+                    analyticsWebClient: Promise.resolve(this.analyticsClientMapper),
                 },
                 options.identifiers,
             );
@@ -159,6 +162,74 @@ export abstract class FeatureFlagClient {
         return gateValue;
     }
 
+    static checkGateValueWithInstrumentation(gate: Features): any {
+        if (this.featureGateOverrides.hasOwnProperty(gate)) {
+            const value = this.featureGateOverrides[gate];
+            featureGateExposureBoolEvent(gate, false, value, 3).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+            return value;
+        }
+
+        let gateValue = false;
+        if (FeatureGates.initializeCompleted()) {
+            // FeatureGates.checkGate returns false if any errors
+            gateValue = FeatureGates.checkGate(gate);
+            featureGateExposureBoolEvent(gate, true, gateValue, 0).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+        } else {
+            featureGateExposureBoolEvent(gate, false, gateValue, 1).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+        }
+
+        Logger.debug(`FeatureGates ${gate} -> ${gateValue}`);
+        return gateValue;
+    }
+
+    static checkExperimentBooleanValueWithInstrumentation(experiment: Experiments): any {
+        // unknown experiment name
+        if (!ExperimentGates.hasOwnProperty(experiment)) {
+            featureGateExposureBoolEvent(experiment, false, false, 2).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+            return undefined;
+        }
+
+        if (this.experimentValueOverride.hasOwnProperty(experiment)) {
+            const value = this.experimentValueOverride[experiment];
+            featureGateExposureBoolEvent(experiment, false, value, 3).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+            return value;
+        }
+
+        const experimentGate = ExperimentGates[experiment];
+        let gateValue = experimentGate.defaultValue;
+        if (FeatureGates.initializeCompleted()) {
+            gateValue = FeatureGates.getExperimentValue(experiment, experimentGate.parameter, 'N/A');
+
+            if (gateValue === 'N/A') {
+                gateValue = experimentGate.defaultValue;
+                featureGateExposureBoolEvent(experiment, false, gateValue, 4).then((e) => {
+                    this.analyticsClient.sendTrackEvent(e);
+                });
+            } else {
+                featureGateExposureBoolEvent(experiment, true, gateValue, 0).then((e) => {
+                    this.analyticsClient.sendTrackEvent(e);
+                });
+            }
+        } else {
+            featureGateExposureBoolEvent(experiment, false, gateValue, 1).then((e) => {
+                this.analyticsClient.sendTrackEvent(e);
+            });
+        }
+
+        Logger.debug(`Experiment ${experiment} -> ${gateValue}`);
+        return gateValue;
+    }
+
     static dispose() {
         FeatureGates.shutdownStatsig();
     }

src/util/featureFlags/features.ts

@@ -1,6 +1,7 @@
 export const enum Features {
     EnableNewUriHandler = 'atlascode-enable-new-uri-handler',
     NewSidebarTreeView = 'atlascode-new-sidebar-treeview',
+    NoOpFeature = 'atlascode-noop',
 }
 
 export const enum Experiments {