Add Monorail telemetry for app config validate

packages/app/src/cli/commands/app/config/validate.test.ts

@@ -1,6 +1,7 @@
 import Validate from './validate.js'
 import {linkedAppContext} from '../../../services/app-context.js'
 import {validateApp} from '../../../services/validate.js'
+import metadata from '../../../metadata.js'
 import {testAppLinked} from '../../../models/app/app.test-data.js'
 import {LocalConfigError} from '../../../models/app/local-config-error.js'
 import {describe, expect, test, vi} from 'vitest'
@@ -9,6 +10,7 @@ import {outputResult} from '@shopify/cli-kit/node/output'
 
 vi.mock('../../../services/app-context.js')
 vi.mock('../../../services/validate.js')
+vi.mock('../../../metadata.js', () => ({default: {addPublicMetadata: vi.fn()}}))
 vi.mock('@shopify/cli-kit/node/output', async (importOriginal) => {
   const actual = await importOriginal<typeof import('@shopify/cli-kit/node/output')>()
   return {
@@ -17,6 +19,13 @@ vi.mock('@shopify/cli-kit/node/output', async (importOriginal) => {
   }
 })
 
+async function expectValidationMetadataCalls(...expectedMetadata: Record<string, unknown>[]) {
+  const metadataCalls = vi.mocked(metadata.addPublicMetadata).mock.calls.map(([getMetadata]) => getMetadata)
+  expect(metadataCalls).toHaveLength(expectedMetadata.length)
+
+  await expect(Promise.all(metadataCalls.map((getMetadata) => getMetadata()))).resolves.toEqual(expectedMetadata)
+}
+
 describe('app config validate command', () => {
   test('calls validateApp with json: false by default', async () => {
     // Given
@@ -29,6 +38,7 @@ describe('app config validate command', () => {
 
     // Then
     expect(validateApp).toHaveBeenCalledWith(app, {json: false})
+    await expectValidationMetadataCalls({cmd_app_validate_json: false})
   })
 
   test('calls validateApp with json: true when --json flag is passed', async () => {
@@ -42,6 +52,7 @@ describe('app config validate command', () => {
 
     // Then
     expect(validateApp).toHaveBeenCalledWith(app, {json: true})
+    await expectValidationMetadataCalls({cmd_app_validate_json: true})
   })
 
   test('calls validateApp with json: true when -j flag is passed', async () => {
@@ -55,6 +66,7 @@ describe('app config validate command', () => {
 
     // Then
     expect(validateApp).toHaveBeenCalledWith(app, {json: true})
+    await expectValidationMetadataCalls({cmd_app_validate_json: true})
   })
 
   test('rethrows LocalConfigError in non-json mode without emitting json', async () => {
@@ -67,6 +79,14 @@ describe('app config validate command', () => {
     await expect(Validate.run(['--path=/tmp/app'], import.meta.url)).rejects.toThrow()
     expect(outputResult).not.toHaveBeenCalled()
     expect(validateApp).not.toHaveBeenCalled()
+    await expectValidationMetadataCalls(
+      {cmd_app_validate_json: false},
+      {
+        cmd_app_validate_valid: false,
+        cmd_app_validate_issue_count: 1,
+        cmd_app_validate_file_count: 1,
+      },
+    )
   })
 
   test('outputs structured configuration issues from app loading before validateApp runs', async () => {
@@ -109,6 +129,14 @@ describe('app config validate command', () => {
       ),
     )
     expect(validateApp).not.toHaveBeenCalled()
+    await expectValidationMetadataCalls(
+      {cmd_app_validate_json: true},
+      {
+        cmd_app_validate_valid: false,
+        cmd_app_validate_issue_count: 1,
+        cmd_app_validate_file_count: 1,
+      },
+    )
   })
 
   test('outputs a root json issue when app loading fails without structured issues', async () => {
@@ -140,6 +168,14 @@ describe('app config validate command', () => {
       ),
     )
     expect(validateApp).not.toHaveBeenCalled()
+    await expectValidationMetadataCalls(
+      {cmd_app_validate_json: true},
+      {
+        cmd_app_validate_valid: false,
+        cmd_app_validate_issue_count: 1,
+        cmd_app_validate_file_count: 1,
+      },
+    )
   })
 
   test('outputs json when validateApp throws a structured configuration abort', async () => {
@@ -181,6 +217,14 @@ describe('app config validate command', () => {
         2,
       ),
     )
+    await expectValidationMetadataCalls(
+      {cmd_app_validate_json: true},
+      {
+        cmd_app_validate_valid: false,
+        cmd_app_validate_issue_count: 1,
+        cmd_app_validate_file_count: 1,
+      },
+    )
   })
 
   test('rethrows non-configuration errors from validateApp in json mode without converting them to validation json', async () => {
@@ -192,6 +236,7 @@ describe('app config validate command', () => {
     // When / Then
     await expect(Validate.run(['--json'], import.meta.url)).rejects.toThrow()
     expect(outputResult).not.toHaveBeenCalled()
+    await expectValidationMetadataCalls({cmd_app_validate_json: true})
   })
 
   test('rethrows unrelated abort errors in json mode without converting them to validation json', async () => {
@@ -202,5 +247,6 @@ describe('app config validate command', () => {
     await expect(Validate.run(['--json', '--path=/tmp/app'], import.meta.url)).rejects.toThrow()
     expect(outputResult).not.toHaveBeenCalled()
     expect(validateApp).not.toHaveBeenCalled()
+    await expectValidationMetadataCalls({cmd_app_validate_json: true})
   })
 })

packages/app/src/cli/commands/app/config/validate.ts

@@ -2,13 +2,18 @@ import {appFlags} from '../../../flags.js'
 import {validateApp} from '../../../services/validate.js'
 import AppLinkedCommand, {AppLinkedCommandOutput} from '../../../utilities/app-linked-command.js'
 import {linkedAppContext} from '../../../services/app-context.js'
+import metadata from '../../../metadata.js'
 import {toRootValidationIssue} from '../../../models/app/error-parsing.js'
 import {LocalConfigError} from '../../../models/app/local-config-error.js'
 import {invalidAppValidationResult, stringifyAppValidationResult} from '../../../services/validation-result.js'
 import {globalFlags, jsonFlag} from '@shopify/cli-kit/node/cli'
 import {AbortSilentError} from '@shopify/cli-kit/node/error'
 import {outputResult, stringifyMessage} from '@shopify/cli-kit/node/output'
 
+function getValidationIssueFileCount(issues: {filePath: string}[]) {
+  return new Set(issues.map((issue) => issue.filePath)).size
+}
+
 export default class Validate extends AppLinkedCommand {
   static summary = 'Validate your app configuration and extensions.'
 
@@ -25,6 +30,10 @@ export default class Validate extends AppLinkedCommand {
   public async run(): Promise<AppLinkedCommandOutput> {
     const {flags} = await this.parse(Validate)
 
+    await metadata.addPublicMetadata(() => ({
+      cmd_app_validate_json: flags.json,
+    }))
+
     try {
       const {app} = await linkedAppContext({
         directory: flags.path,
@@ -38,13 +47,22 @@ export default class Validate extends AppLinkedCommand {
 
       return {app}
     } catch (error) {
-      if (flags.json && error instanceof LocalConfigError) {
+      if (error instanceof LocalConfigError) {
         const issues =
           error.issues.length > 0
             ? error.issues
             : [toRootValidationIssue(error.configurationPath, stringifyMessage(error.message).trim())]
-        outputResult(stringifyAppValidationResult(invalidAppValidationResult(issues)))
-        throw new AbortSilentError()
+
+        await metadata.addPublicMetadata(() => ({
+          cmd_app_validate_valid: false,
+          cmd_app_validate_issue_count: issues.length,
+          cmd_app_validate_file_count: getValidationIssueFileCount(issues),
+        }))
+
+        if (flags.json) {
+          outputResult(stringifyAppValidationResult(invalidAppValidationResult(issues)))
+          throw new AbortSilentError()
+        }
       }
 
       throw error

packages/app/src/cli/services/validate.test.ts

@@ -1,11 +1,13 @@
 import {validateApp} from './validate.js'
 import {testAppLinked} from '../models/app/app.test-data.js'
+import metadata from '../metadata.js'
 import {AppErrors} from '../models/app/loader.js'
 import {describe, expect, test, vi} from 'vitest'
 import {outputResult} from '@shopify/cli-kit/node/output'
 import {renderError, renderSuccess} from '@shopify/cli-kit/node/ui'
 import {AbortSilentError} from '@shopify/cli-kit/node/error'
 
+vi.mock('../metadata.js', () => ({default: {addPublicMetadata: vi.fn()}}))
 vi.mock('@shopify/cli-kit/node/output', async (importOriginal) => {
   const actual = await importOriginal<typeof import('@shopify/cli-kit/node/output')>()
   return {
@@ -15,6 +17,16 @@ vi.mock('@shopify/cli-kit/node/output', async (importOriginal) => {
 })
 vi.mock('@shopify/cli-kit/node/ui')
 
+async function expectLastValidationMetadata(expected: {
+  cmd_app_validate_valid: boolean
+  cmd_app_validate_issue_count: number
+  cmd_app_validate_file_count: number
+}) {
+  const getMetadata = vi.mocked(metadata.addPublicMetadata).mock.calls.at(-1)?.[0]
+  expect(getMetadata).toBeDefined()
+  await expect(Promise.resolve(getMetadata!())).resolves.toEqual(expected)
+}
+
 describe('validateApp', () => {
   test('renders success when there are no errors', async () => {
     // Given
@@ -27,6 +39,11 @@ describe('validateApp', () => {
     expect(renderSuccess).toHaveBeenCalledWith({headline: 'App configuration is valid.'})
     expect(renderError).not.toHaveBeenCalled()
     expect(outputResult).not.toHaveBeenCalled()
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: true,
+      cmd_app_validate_issue_count: 0,
+      cmd_app_validate_file_count: 0,
+    })
   })
 
   test('outputs json success when --json is enabled and there are no errors', async () => {
@@ -40,6 +57,11 @@ describe('validateApp', () => {
     expect(outputResult).toHaveBeenCalledWith(JSON.stringify({valid: true, issues: []}, null, 2))
     expect(renderSuccess).not.toHaveBeenCalled()
     expect(renderError).not.toHaveBeenCalled()
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: true,
+      cmd_app_validate_issue_count: 0,
+      cmd_app_validate_file_count: 0,
+    })
   })
 
   test('renders errors and throws when there are validation errors', async () => {
@@ -58,6 +80,11 @@ describe('validateApp', () => {
     })
     expect(renderSuccess).not.toHaveBeenCalled()
     expect(outputResult).not.toHaveBeenCalled()
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: false,
+      cmd_app_validate_issue_count: 2,
+      cmd_app_validate_file_count: 2,
+    })
   })
 
   test('outputs json errors and throws when --json is enabled and there are validation errors', async () => {
@@ -95,6 +122,11 @@ describe('validateApp', () => {
     )
     expect(renderError).not.toHaveBeenCalled()
     expect(renderSuccess).not.toHaveBeenCalled()
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: false,
+      cmd_app_validate_issue_count: 2,
+      cmd_app_validate_file_count: 2,
+    })
   })
 
   test('outputs only structured issues when the rendered message matches them exactly', async () => {
@@ -221,6 +253,36 @@ describe('validateApp', () => {
         2,
       ),
     )
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: false,
+      cmd_app_validate_issue_count: 1,
+      cmd_app_validate_file_count: 1,
+    })
+  })
+
+  test('counts a distinct root issue when the rendered message changes to a new same-file error', async () => {
+    // Given
+    const errors = new AppErrors()
+    errors.addError('/path/to/shopify.app.toml', '• [client_id]: Required', [
+      {
+        filePath: '/path/to/shopify.app.toml',
+        path: ['client_id'],
+        pathString: 'client_id',
+        message: 'Required',
+        code: 'invalid_type',
+      },
+    ])
+    errors.addError('/path/to/shopify.app.toml', 'Unsupported section(s) in app configuration: webhooks')
+    const app = testAppLinked()
+    app.errors = errors
+
+    // When / Then
+    await expect(validateApp(app, {json: true})).rejects.toThrow(AbortSilentError)
+    await expectLastValidationMetadata({
+      cmd_app_validate_valid: false,
+      cmd_app_validate_issue_count: 2,
+      cmd_app_validate_file_count: 1,
+    })
   })
 
   test('renders success when errors object exists but is empty', async () => {