From 88ab981ec9e8ffe96b543d14359c7aa5e93f0a8e Mon Sep 17 00:00:00 2001 From: brock-statsig <146870406+brock-statsig@users.noreply.github.com> Date: Thu, 26 Dec 2024 16:59:56 -0600 Subject: [PATCH] deletes and redirects (#2443) --- .../experimentation/best-practices.md | 7 -- .../choosing-randomization-unit.md | 88 ------------------- .../experimentation/common-terms.md | 13 --- .../experimentation/scenarios.md | 24 ----- .../experimentation/why-experiment.md | 11 --- docs/integrations/data-connectors/segment.mdx | 2 +- docs/metrics/different-id.md | 2 +- .../guides/running_a_poc.mdx | 2 +- docusaurus.config.ts | 16 ++++ 9 files changed, 19 insertions(+), 146 deletions(-) delete mode 100644 docs/experiments-plus/experimentation/best-practices.md delete mode 100644 docs/experiments-plus/experimentation/choosing-randomization-unit.md delete mode 100644 docs/experiments-plus/experimentation/common-terms.md delete mode 100644 docs/experiments-plus/experimentation/scenarios.md delete mode 100644 docs/experiments-plus/experimentation/why-experiment.md diff --git a/docs/experiments-plus/experimentation/best-practices.md b/docs/experiments-plus/experimentation/best-practices.md deleted file mode 100644 index 5ddd571c9..000000000 --- a/docs/experiments-plus/experimentation/best-practices.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: Best Practices -sidebar_label: Best Practices -slug: /experiments-plus/experimentation/best-practices ---- - -For a good overview on Experiment Design, Monitoring and Readout, see [this article](https://statsig.com/blog/product-experimentation-best-practices). diff --git a/docs/experiments-plus/experimentation/choosing-randomization-unit.md b/docs/experiments-plus/experimentation/choosing-randomization-unit.md deleted file mode 100644 index 1e1b23c60..000000000 --- a/docs/experiments-plus/experimentation/choosing-randomization-unit.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: Choosing the Randomization Unit -sidebar_label: Choosing the Randomization Unit -slug: /experiments-plus/experimentation/choosing-randomization-unit ---- - -# Choosing the Unit of Randomization -When designing an experiment, you will pick a **unit of randomization** to decide *who* or *what* is randomly allocated to your control and treatment groups. -The choice of the randomization unit is critical in experiment design as it impacts your user experience as well as the accuracy of your experiment's results. -Choosing the right randomization unit will ensure that you deliver a consistent experience to your users and get reliable results from your experiments. - -## Some units are better than others -A key assumption in most A/B tests and experiments is the **stable unit treatment value assumption (SUTVA)**, where the response of a unit of randomization under treatment is -independent of the response of another unit under treatment. The most common unit of randomization is the user identifier your application uses to uniquely identify an individual user. -You may choose to use other types of identifiers based on the kind of experiment you want to run and the constraints around it as outlined below. - -### User Identifiers -**Registered User IDs** are the most commonly used units of randomization. Your application would generally create a registered user ID after the user has registered with your application and created an account. -Available as long as the user stays signed-in, the user ID is the most popular unit of randomization as it ensures a consistent user experience, -across sessions and devices. It doesn't depend on client-side cookies, which may be cleared by the user at any time. - ---- -**Learn More** - -You can supply a user ID as part of the ``StatsigUser`` object when you implement an feature gate or experiment on Statsig. See [Statsig Docs](/server/concepts/user) to learn more. - ---- - -### Other Stable Identifiers -**Device IDs** or **Anonymous User IDs** are used as units of randomization for experiments that involve users who haven't registered or signed into your application. -For example, you may choose to use a device ID or an anonymous user ID when you want to test the impact of different landing page options on user registration. -As the device is a stable vehicle of usage for the user, it offers a stable identifier to observe the user's behavior over their journey with your application. -As a variant of this approach, some applications may choose identify anonymous users by saving first party cookies on the user's device. - -#### Drawbacks - While these identifiers offer a stable tracking mechanism, they do have certain drawbacks. - - The main drawback is that you won't have access to this identifier if the same user engages with your application on a different device. - - A less common drawback arises when multiple users may use the same device. In this case, you will end up including their combined engagement in the metrics you use to evaluate experiments. - - As these identifiers are device-specific, they are available only with client SDKs to help you instrument the client-side of your application. These identifiers are not available with server SDKs. - - ---- -**Learn More** - -- Read more about [User-level vs. Device-level experiments](https://blog.statsig.com/user-level-vs-device-level-experiments-with-statsig-338d48a81778) and how these identifiers are used to report the right experiment results. -- Statsig client SDKs automatically generate **Stable IDs** for your users when you choose to run a device-level experiment. See the [Statsig Guide for Device Experiments](../../guides/first-device-level-experiment) to learn more about how to use stable IDs for experiments involving anonymous users. - ---- - -### Other Identifiers -**Session IDs** are used in select use cases where the metric you're trying to improve is meaningful within a session *and* when you can safely assume that each session is independent. -An example where you may choose to use session IDs when running experiments to optimize conversion rate for guest checkouts that are tracked on a per session basis. - Another use case for sessions IDs is when you need an identifier for use with server SDKs but want to run experiments for users who haven't yet unregistered or have signed-out. - -#### Drawbacks -As users frequently remember their experience from an earlier session, assuming user sessions to be independent can be a significant assumption to make for most experiments. -For example, if a user sees a product in one session as part of the control group and returns to complete the purchase in a different session, there's no guarantee they'll be placed in the control group again. -If this time they're placed in the treatment group, you may overestimate the positive impact of the treatment. - - -## Using multiple identifiers -When you're running multiple experiments, you may choose to use a different identifier for each experiment depending on the context. -Consider a scenario where you're running two experiments as shown below. One experiment (A) tracks the impact of a new mobile registration flow on the number of user registrations. -Another experiment (B) tracks the impact of a new upgrade flow for converting your registered users to subscribed users. -Ideally, you also want to track the how your new mobile registration flow impacts downstream conversion to subscribed users. - -In this scenario, experiment A will require an identifier that you can use over the entire user journey, say a stable device ID. -For experiment B, you may prefer to use the user ID that forms the basis for most of your existing business metrics such as the rate of conversion to your subscription products. - - -![Device Level Experiments](https://user-images.githubusercontent.com/74588208/141707011-95c0c859-c60f-45f8-a6da-d31664f05e06.png) - - - - - - - - - - - - - - - - - diff --git a/docs/experiments-plus/experimentation/common-terms.md b/docs/experiments-plus/experimentation/common-terms.md deleted file mode 100644 index 6315c5a69..000000000 --- a/docs/experiments-plus/experimentation/common-terms.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: Common Terms -sidebar_label: Common Terms -slug: /experiments-plus/experimentation/common-terms ---- - -- A **Control Variable** is an experimental variable that is thought to influence the key metrics of interest. In simple A/B experiments, a single control variable is assigned two values. It is more common to assign multiple values such as A,B,C, or D to a single control variable. Multivariate experiments evaluate multiple control variables that allow experimenters to discover a global optimum when multiple variables interact. -- A **Variant** is a product or feature experience being tested, often by assigning values to control variables. In a simple A/B experiments, A and B are two variants, usually called Control and Treatment. -- A **Randomization Unit** is the most granular unit that can participate in an experiment. Each eligible unit is randomly assigned to a variant, allowing causality to be determined with high probability. It is very common to use users as a randomization unit and Statsig highly recommends using users for running controlled experiments. -- **Statistical Significance** can be assessed using multiple approaches. Two of these approaches are using the p-value and the confidence interval: - - The **p-value** measures the probability of the metric lift you observe (or a more extreme lift) assuming that the variant you’re testing has no effect. The standard is to use a p-value less than 0.05 to identify variants that have a statistically significant effect. A p-value less than 0.05 implies that there’s less than 5% chance of seeing the observed metric lift (or a more extreme metric lift) if the variant had no effect. In practice, a p-value that's lower than your pre-defined threshold is treated as evidence for there being a true effect. - - A **confidence interval** examines whether the metric difference between the variant and control overlaps with zero. A 95% confidence interval is the range that covers the true difference 95% of the time. It is usually centered around the observed delta between the variant and control with an extension of 1.96 standard errors on each side. - diff --git a/docs/experiments-plus/experimentation/scenarios.md b/docs/experiments-plus/experimentation/scenarios.md deleted file mode 100644 index 59e3ffa22..000000000 --- a/docs/experiments-plus/experimentation/scenarios.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Scenarios -sidebar_label: Scenarios -slug: /experiments-plus/experimentation/scenarios ---- - -Statsig sees two broad scenarios for experimentation. - -## 1. Experiment to grow faster - -Experiments can help climb a hill to a local optimum based on your current business strategy and product portfolio. For example, -- Experiments can optimize for the ideal user experience for a given functionality -- Experiments can help iterate on the functionality, algorithms, and infrastructure that matter the most to your users and your business -- Experiments can identify proposals with the highest return for effort required - -Identifying the metrics that both reflect your strategic direction and are sensitive to the changes you make ensures that you don’t waste resources. Identifying guardrail metrics that you want to hold regardless of the changes you make compels explicit tradeoffs and prevents you from regressing on the fundamental needs of your business. - -## 2. Experiment to discover faster - -Experiments can help develop a portfolio of ideas that may point to a larger hill or opportunity. Navigating these bigger jumps may require: -- Experiments that run for a longer duration to mitigate any novelty effects and to ensure that you have given the new product version enough time to build adoption -- Experiments that ramp slowly and progressively to more users to limit risk and to build more statistical power before launch -- Many different experiments that test several related hypotheses that form a new business strategy - diff --git a/docs/experiments-plus/experimentation/why-experiment.md b/docs/experiments-plus/experimentation/why-experiment.md deleted file mode 100644 index ebe953357..000000000 --- a/docs/experiments-plus/experimentation/why-experiment.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Why Experiment -sidebar_label: Why Experiment -slug: /experiments-plus/experimentation/why-experiment ---- - -Controlled experiments are the best scientific way to establish causality between your product features and customer impact. Establishing such causality allows you to only ship features that improve customer experience. This can make experiments the driving force behind your **pace of innovation**. - -As you grow your pace of innovation, experiments also enable you to also measure the success of the features you ship and uncover unexpected side effects with every code change. This allows you to iterate faster in the short term, establish key business drivers, and **make better, evidence-driven business decisions every day**. - -In comparison, relationships observed in historical metrics cannot be considered structural or causal because multiple uncaptured external and internal factors influence customer behavior. Historical metrics establish correlation, not causation. diff --git a/docs/integrations/data-connectors/segment.mdx b/docs/integrations/data-connectors/segment.mdx index 0cd94b90d..fae275c26 100644 --- a/docs/integrations/data-connectors/segment.mdx +++ b/docs/integrations/data-connectors/segment.mdx @@ -93,7 +93,7 @@ If you are unable to connect to Segment via OAuth, you can still manually connec ![](https://user-images.githubusercontent.com/1315028/150830169-17564060-816b-4c5c-ade9-10bf6274265a.png) ## Working with Users -Statsig will join incoming user identifiers to whichever [unit of randomization](/experiments-plus/experimentation/choosing-randomization-unit) you choose. This allows you to be flexible with your experimentation and enables testing on known (userID) and unknown (anonymousID) traffic as well as any custom identifiers your team may have (deviceID, companyID, vehicleID, etc). +Statsig will join incoming user identifiers to whichever [unit of randomization](/experiments-plus#choosing-the-right-randomization-unit) you choose. This allows you to be flexible with your experimentation and enables testing on known (userID) and unknown (anonymousID) traffic as well as any custom identifiers your team may have (deviceID, companyID, vehicleID, etc). ### User IDs and Custom IDs diff --git a/docs/metrics/different-id.md b/docs/metrics/different-id.md index 230aaff3d..e3469894c 100644 --- a/docs/metrics/different-id.md +++ b/docs/metrics/different-id.md @@ -10,7 +10,7 @@ There are two common scenarios where the experiment assignment unit differs from 1. Measuring session-level metrics for a user-level experiment. Ratio metrics are commonly used to solve this (this doc). 2. Measuring logged-in metrics (eg. revenue) on a logged-out experiment. There are two solutions: - a. Running the experiment at the [device-level](/experiments-plus/experimentation/choosing-randomization-unit#other-stable-identifiers), with device-level metrics collected even after the user is logged-in. + a. Running the experiment at the [device-level](/guides/first-device-level-experiment), with device-level metrics collected even after the user is logged-in. b. Using [ID resolution](/statsig-warehouse-native/features/id-resolution). We will explain how to set up the first scenario with Warehouse Native in this doc. diff --git a/docs/statsig-warehouse-native/guides/running_a_poc.mdx b/docs/statsig-warehouse-native/guides/running_a_poc.mdx index 444bba58e..5f6eefea9 100644 --- a/docs/statsig-warehouse-native/guides/running_a_poc.mdx +++ b/docs/statsig-warehouse-native/guides/running_a_poc.mdx @@ -27,7 +27,7 @@ Keep these high level steps in mind as you begin your planning your Warehouse Na - This approach can yield results for analysis in as little as **30 minutes,** assuming data is readily available for ingestion - If your team plans on utilizing the **Assign and Analyze** experimentation option, you’ll want to identify **where** the experiment will run. Typically **web based** experiments are easier to evaluate, however Statsig has SDK support for server and mobile SDKs as well. - **Note**: It’s important the implementing team understands how the SDKs operate prior to executing a proof of concept. Our [client](/client/introduction) and [server](/server/introduction) docs can help orient your team! - - A typical evaluation takes **2-4 weeks** to account for experiment design, implementation, time to bake, and analysis. To ensure a successful POC, [have a well scoped plan](/guides/running-a-poc#phase-0-scope--prepare-your-poc) and ensure the right teams are included to assist along the way. + - A typical evaluation takes **2-4 weeks** to account for experiment design, implementation, time to bake, and analysis. To ensure a successful POC, [have a well scoped plan](/guides/running-a-poc#2-phase-0-scope--prepare-your-poc) and ensure the right teams are included to assist along the way. - Read [experimentation best practices](https://statsig.com/blog/product-experimentation-best-practices) to get an idea of how to best succeed. 1. **Connect the Warehouse** - In order to query data and operate within your warehouse, you’ll need to allocate resources and connect to Statsig. You may choose to utilize an existing prod database or create a separate cluster specifically for experimentation (if you don’t already have one). diff --git a/docusaurus.config.ts b/docusaurus.config.ts index d66c77fea..a9585acfa 100644 --- a/docusaurus.config.ts +++ b/docusaurus.config.ts @@ -144,6 +144,22 @@ const config: Config = { "@docusaurus/plugin-client-redirects", { redirects: [ + { + from: "/experiments-plus/experimentation/why-experiment", + to: "/experiments-plus#why-experiment", + }, + { + from: "/experiments-plus/experimentation/scenarios", + to: "/experiments-plus#scenarios-for-experimentation", + }, + { + from: "/experiments-plus/experimentation/common-terms", + to: "/experiments-plus#key-concepts-in-experimentation", + }, + { + from: "/experiments-plus/experimentation/choosing-randomization-unit", + to: "/experiments-plus#choosing-the-right-randomization-unit", + }, { from: "/js-migration", to: "/client/javascript-sdk/migrating-from-statsig-js",