Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Token extraction updates #29846

Merged
merged 2 commits into from
Oct 9, 2024
Merged

Token extraction updates #29846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
46c96de
build: extract correct tokens set from core theme
crisbeto Oct 9, 2024
a869e1a
build: account for overlapping names when extracting tokens
crisbeto Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/material/core/_core-theme.scss
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,19 @@
}
}

@mixin overrides($tokens: ()) {
// This theme is a special case where not all of the imported tokens are supported in `overrides`.
// To aid the docs token extraction, we have to pull the `overrides` token config out into a
// separate function.
// !!!Important!!! renaming or removal of this function requires the `extract-tokens.ts` script to
// be updated as well.
@function _get-supported-overrides-tokens() {
$app-tokens: tokens-mat-app.get-token-slots();
$ripple-tokens: tokens-mat-ripple.get-token-slots();
$option-tokens: tokens-mat-option.get-token-slots();
$full-pseudo-checkbox-tokens: tokens-mat-full-pseudo-checkbox.get-token-slots();
$minimal-pseudo-checkbox-tokens: tokens-mat-minimal-pseudo-checkbox.get-token-slots();

@include token-utils.batch-create-token-values(
$tokens,
@return (
(prefix: tokens-mat-app.$prefix, tokens: $app-tokens),
(prefix: tokens-mat-ripple.$prefix, tokens: $ripple-tokens),
(prefix: tokens-mat-option.$prefix, tokens: $option-tokens),
Expand All @@ -88,6 +92,10 @@
);
}

@mixin overrides($tokens: ()) {
@include token-utils.batch-create-token-values($tokens, _get-supported-overrides-tokens()...);
}

// Mixin that renders all of the core styles that depend on the theme.
@mixin theme($theme, $options...) {
// Wrap the sub-theme includes in the duplicate theme styles mixin. This ensures that
Expand Down
94 changes: 71 additions & 23 deletions tools/extract-tokens/extract-tokens.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,15 @@ import {compileString} from 'sass';
interface Token {
/** Name of the token. */
name: string;
/** Prefix under which the token is exposed in the DOM. */
prefix: string;
/** Type of the token (color, typography etc.) */
type: string;
/** System token that it was derived from. */
derivedFrom?: string;
/** Places from which the token with the specific name can originate. */
sources: {
/** Prefix of the source. */
prefix: string;
/** System-level token from which the source dervies its value. */
derivedFrom?: string;
}[];
}

/** Extracted map of tokens from the source Sass files. */
Expand Down Expand Up @@ -41,11 +44,17 @@ if (require.main === module) {
throw new Error(`Could not find theme files in ${packagePath}`);
}

const themes: {name: string; tokens: Token[]}[] = [];
const themes: {name: string; overridesMixin: string; tokens: Token[]}[] = [];

themeFiles.forEach(theme => {
const tokens = extractTokens(theme.filePath);
themes.push({name: theme.mixinPrefix, tokens});
themes.push({
name: theme.mixinPrefix,
// This can be derived from the `name` already, but we want the source
// of truth to be in this repo, instead of whatever page consumes the data.
overridesMixin: `${theme.mixinPrefix}-overrides`,
tokens,
});
});

writeFileSync(outputPath, JSON.stringify(themes));
Expand Down Expand Up @@ -125,25 +134,46 @@ function extractTokens(themePath: string): Token[] {
* @param groups Extracted data from the Sass file.
*/
function createTokens(type: string, groups: ExtractedTokenMap): Token[] {
const result: Token[] = [];
const data: Map<string, Map<string, string | null>> = new Map();

// First step is to go through the whole data and group the tokens by their name. We need to
// group the tokens, because the same name can be used by different prefixes (e.g. both
// `mdc-filled-text-field` and `mdc-outlined-text-field` have a `label-text-color` token).
Object.keys(groups).forEach(prefix => {
const tokens = groups[prefix];

// The token data for some components may be null.
if (tokens) {
Object.keys(tokens).forEach(name => {
let tokenData = data.get(name);

if (!tokenData) {
tokenData = new Map();
data.set(name, tokenData);
}

const value = tokens[name];
const derivedFrom = typeof value === 'string' ? textBetween(value, 'var(', ')') : null;
const token: Token = {name, prefix, type};
if (derivedFrom) {
token.derivedFrom = derivedFrom.replace('--sys-', '');
}
result.push(token);
tokenData.set(prefix, derivedFrom ? derivedFrom.replace('--sys-', '') : null);
});
}
});

// After the tokens have been grouped, we can create the `Token` object for each one.
const result: Token[] = [];

data.forEach((tokenData, name) => {
const token: Token = {name, type, sources: []};

tokenData.forEach((derivedFrom, prefix) => {
// Set `derivedFrom` to `undefined` if it hasn't been set so it doesn't show up in the JSON.
token.sources.push({prefix, derivedFrom: derivedFrom || undefined});
});

result.push(token);
});

// Sort the tokens by name so they're easier to scan.
return result.sort((a, b) => a.name.localeCompare(b.name));
}

Expand Down Expand Up @@ -187,13 +217,35 @@ function getTokenExtractionCode(

// Goes through all the available namespaces looking for a `$prefix` variable. This allows us to
// determine the prefixes that are used within the theme. Once we know the prefixes we can use
// them to extract only the tokens we care about from the full token set.
// them to extract only the tokens we care about from the full token set. Note: `core-theme`
// is a special case where not all of the imported tokens are supported in the `overrides`
// mixin. Such cases will expose a `_get-supported-overrides-tokens` private function that
// can be used to determine the exact set of prefixes that are used.
// After the tokens are extracted, we log them out using `@debug` and then intercept the log
// in order to get the raw data. We have to go through `@debug`, because there's no way to
// output a Sass map to CSS.
// The tokens are encoded as JSON so we don't have to implement parsing of Sass maps in TS.
const append = `
$__namespaces: (${useStatements.map(stmt => `"${extractNamespace(stmt)}"`).join(', ')});
$__prefixes: ();

@if ${meta}.function-exists('_get-supported-overrides-tokens') {
$__configs: _get-supported-overrides-tokens();

@each $config in $__configs {
$__prefixes: ${list}.append($__prefixes, ${map}.get($config, prefix));
}
} @else {
$__namespaces: (${useStatements.map(stmt => `"${extractNamespace(stmt)}"`).join(', ')});

@each $name in $__namespaces {
$prefix: ${map}.get(${meta}.module-variables($name), prefix);

@if ($prefix) {
$__prefixes: ${list}.append($__prefixes, $prefix);
}
}
}

$__all-color: ${m3Tokens}.generate-color-tokens(light, ${palettes}.$azure-palette,
${palettes}.$azure-palette, ${palettes}.$azure-palette, 'sys');
$__all-typography: ${m3Tokens}.generate-typography-tokens(font, 100, 100, 100, 100, 'sys');
Expand All @@ -204,15 +256,11 @@ function getTokenExtractionCode(
$__density: ();
$__base: ();

@each $name in $__namespaces {
$prefix: map-get(${meta}.module-variables($name), 'prefix');

@if ($prefix) {
$__color: ${map}.set($__color, $prefix, map-get($__all-color, $prefix));
$__typography: ${map}.set($__typography, $prefix, map-get($__all-typography, $prefix));
$__density: ${map}.set($__density, $prefix, map-get($__all-density, $prefix));
$__base: ${map}.set($__base, $prefix, map-get($__all-base, $prefix));
}
@each $prefix in $__prefixes {
$__color: ${map}.set($__color, $prefix, ${map}.get($__all-color, $prefix));
$__typography: ${map}.set($__typography, $prefix, ${map}.get($__all-typography, $prefix));
$__density: ${map}.set($__density, $prefix, ${map}.get($__all-density, $prefix));
$__base: ${map}.set($__base, $prefix, ${map}.get($__all-base, $prefix));
}

// Define our JSON.stringify implementation so it can be used below.
Expand Down
Loading