Merge pull request #4342 from hayemaxi/restructure3

refactor: move toolkit to monorepo structure

These commits restructure our project into a monorepo using npm workspaces. The original toolkit extension code now lives under packages/toolkit/. Additional extensions and common modules will go under the packages/ folder. This change will allow us to divide the toolkit into separate modules and extensions as needed.

A summary of changes include:
- Creating a root package.json
- Moving all extension src code, resources/, types/, and syntaxes to a subproject with the original package.json
- Updating paths throughout the code and configs to reference the new file structure
- Script and packaging changes to work with this new structure

See `docs/ARCHITECTURE.md#monorepo-structure` for more info.

.eslintignore

@@ -1,11 +1,19 @@
-node_modules/**
-dist/**
+node_modules/
+dist/
 src.gen/**
 **/*.json
+**/*.gen.ts
+types/*.d.ts
+
+# Duplicate entries because prettier can be ran from root or within the workspace/subpackage.
+# TODO: Avoid this.
 src/shared/telemetry/clienttelemetry.d.ts
 src/codewhisperer/client/codewhispererclient.d.ts
 src/codewhisperer/client/codewhispereruserclient.d.ts
 src/amazonqFeatureDev/client/featuredevproxyclient.d.ts
-**/*.gen.ts
 src/testFixtures/**
-types/*.d.ts
+packages/toolkit/src/shared/telemetry/clienttelemetry.d.ts
+packages/toolkit/src/codewhisperer/client/codewhispererclient.d.ts
+packages/toolkit/src/codewhisperer/client/codewhispereruserclient.d.ts
+packages/toolkit/src/amazonqFeatureDev/client/featuredevproxyclient.d.ts
+packages/toolkit/src/testFixtures/**

.eslintrc.js

@@ -4,7 +4,7 @@ module.exports = {
     parserOptions: {
         ecmaVersion: 'latest',
         sourceType: 'module',
-        project: './tsconfig.json',
+        project: ['./packages/*/tsconfig.json', './plugins/*/tsconfig.json'],
         tsconfigRootDir: __dirname,
     },
     env: {

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 * @aws/aws-ides-team
-src/codewhisperer/ @aws/codewhisperer-team
-src/amazonqFeatureDev/ @aws/earlybird
-src/codewhispererChat/ @aws/aws-mynah
-src/amazonq/ @aws/aws-mynah
+packages/toolkit/src/codewhisperer/ @aws/codewhisperer-team
+packages/toolkit/src/amazonqFeatureDev/ @aws/earlybird
+packages/toolkit/src/codewhispererChat/ @aws/aws-mynah
+packages/toolkit/src/amazonq/ @aws/aws-mynah

.github/dependabot.yml

@@ -7,7 +7,7 @@
 version: 2
 updates:
     - package-ecosystem: 'npm'
-      directory: '/' # Location of package manifests.
+      directory: './packages/toolkit/' # Location of package manifests.
       target-branch: 'master' # Avoid updates to "staging".
       commit-message:
           prefix: 'deps'
@@ -25,7 +25,7 @@ updates:
                   - '@smithy*'
                   - 'smithy*'
     - package-ecosystem: 'github-actions'
-      directory: '/'
+      directory: './packages/toolkit/'
       target-branch: 'master' # Avoid updates to "staging".
       commit-message:
           prefix: 'deps'

.github/workflows/release.yml

@@ -52,7 +52,7 @@ jobs:
             - name: vsix
               run: |
                   npm run createRelease  # Generate CHANGELOG.md
-                  npm run generateNonCodeFiles
+                  npm run generateNonCodeFiles -w packages/toolkit
                   cp ./README.quickstart.vscode.md ./README.md
                   npm run package -- --feature "$FEAT_NAME"
             - uses: actions/upload-artifact@v4

.gitignore

@@ -1,8 +1,8 @@
 out
 dist
 node_modules
-.vscode-test/
-coverage/
+.vscode-test
+coverage
 *.vsix
 *.zip
 *.bk
@@ -13,34 +13,40 @@ quickStart*.html
 README.quickstart.cloud9.md
 README.quickstart.vscode.md
 .gitcommit
-resources/debugger/__pycache__
+__pycache__
 
 # Generated if running the `depcruise` command from the documentation example
 /dependency-graph.svg
 
 # Auto generated definitions
-src/**/*.gen.ts
+packages/toolkit/src/**/*.gen.ts
 src.gen/*
 # Telemetry definition for testing adding telemetry
-src/shared/telemetry/vscodeTelemetry.json
+packages/toolkit/src/shared/telemetry/vscodeTelemetry.json
 
 # Test reports
-.test-reports/
+.test-reports
 
 # Auto generated type definitions
-src/shared/telemetry/clienttelemetry.d.ts
-src/codewhisperer/client/codewhispererclient.d.ts
-src/codewhisperer/client/codewhispereruserclient.d.ts
-src/amazonqFeatureDev/client/featuredevproxyclient.d.ts
+packages/toolkit/src/shared/telemetry/clienttelemetry.d.ts
+packages/toolkit/src/codewhisperer/client/codewhispererclient.d.ts
+packages/toolkit/src/codewhisperer/client/codewhispereruserclient.d.ts
+packages/toolkit/src/amazonqFeatureDev/client/featuredevproxyclient.d.ts
 
 # Generated by tests
-src/testFixtures/**/bin
-src/testFixtures/**/obj
+packages/toolkit/src/testFixtures/**/bin
+packages/toolkit/src/testFixtures/**/obj
+
+# Generated by copyFiles.ts
+packages/toolkit/README*
+packages/toolkit/CHANGELOG.md
+packages/toolkit/LICENSE
+packages/toolkit/NOTICE
 
 # Icons
-resources/icons/cloud9/generated/**
-resources/fonts/aws-toolkit-icons.woff
-resources/css/icons.css
+packages/toolkit/resources/icons/cloud9/generated/**
+packages/toolkit/resources/fonts/aws-toolkit-icons.woff
+packages/toolkit/resources/css/icons.css
 
 # local configuration
 .local.env

.prettierignore

@@ -1,12 +1,15 @@
 out
-resources/endpoints.json
 CHANGELOG.md
-src/shared/telemetry/service-2.json
 .changes
-src/testFixtures/**
 *.d.ts
 *.gen.ts
-dist/**
-types/*.d.ts
+dist
 src.gen/**
-plugins/*/dist/**
+
+# Duplicate entries because prettier can be ran from root or within the workspace/subpackage.
+# TODO: Avoid this.
+src/shared/telemetry/service-2.json
+src/testFixtures/**
+
+packages/toolkit/src/shared/telemetry/service-2.json
+packages/toolkit/src/testFixtures/**

.vscode/settings.json

@@ -11,5 +11,6 @@
     },
     "typescript.preferences.importModuleSpecifier": "relative",
     "javascript.preferences.importModuleSpecifier": "relative",
-    "typescript.tsdk": "node_modules/typescript/lib"
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "npm.packageManager": "npm"
 }

CONTRIBUTING.md

@@ -8,6 +8,10 @@ codebase and sending pull requests.
 
 ## Getting Started
 
+This project is set up as a typescript monorepo. The documentation throughout this project
+is referring to the subproject in [`packages/toolkit/`](./packages/toolkit/). For more information,
+see [ARCHITECTURE.md](./docs/ARCHITECTURE.md#monorepo-structure)
+
 ### Find things to do
 
 If you're looking for ideas about where to contribute, consider
@@ -35,6 +39,9 @@ Then clone the repository and install NPM packages:
 
 ### Run
 
+Due to the monorepo structure of the project, you can run the extension in VSCode by either opening the
+`aws-toolkit-vscode/packages/toolkit` folder directly, or adding it as a root folder in the VSCode Workspace.
+
 To run the extension from VSCode as a Node.js app:
 
 1. Select the Run panel from the sidebar.
@@ -153,7 +160,9 @@ See [browser.md](./docs/browser.md) for working with the browser implementation
 See [TESTPLAN.md](./docs/TESTPLAN.md) to understand the project's test
 structure, mechanics and philosophy.
 
-You can run tests directly from VSCode:
+You can run tests directly from VSCode. Due to the monorepo structure of the project, you must either open
+the `aws-toolkit-vscode/packages/toolkit/` folder directly, or add it as a root folder in the VSCode Workspace.
+Then:
 
 1. Select `View > Debug`, or select the Debug pane from the sidebar.
 2. From the dropdown at the top of the Debug pane, select the `Extension Tests` configuration.
@@ -208,7 +217,7 @@ To run tests against a specific folder in VSCode, do any one of:
 
 Running the extension in the browser (eg: [vscode.dev](https://vscode.dev/)).
 
-[See documentation here](./src/browser/README.md).
+[See documentation here](./packages/toolkit/src/browser/README.md).
 
 ### Coverage report
 

buildspec/packageTestVsix.yml

@@ -27,7 +27,7 @@ phases:
             - export HOME=/home/codebuild-user
             # Generate CHANGELOG.md
             - npm run createRelease
-            - npm run generateNonCodeFiles
+            - npm run generateNonCodeFiles -w packages/toolkit
             - cp ./README.quickstart.vscode.md ./README.md
             - npm run package
 

codecov.yml

@@ -38,7 +38,7 @@ github_checks:
 flags:
     codewhisperer:
         paths:
-            - src/codewhisperer/
+            - packages/toolkit/src/codewhisperer/
     amazonqFeatureDev:
         paths:
-            - src/amazonqFeatureDev/
+            - packages/toolkit/src/amazonqFeatureDev/

designs/credentials/credentials-management.md

@@ -31,11 +31,11 @@ A formatted version of the Credentials Provider Id may be surfaced to users, how
 
 When the user connects to AWS in the Toolkit, a Credentials Provider is requested, which is then used to obtain credentials. The toolkit requests a Credentials Provider by checking which credentials provider factories support the provider's credentials type. The factories of interest are queried to see which (if any) have the requested Credentials Provider.
 
-At the time this document was written, Shared Credentials are the only supported Credentials. Additional credentials providers will reside at [/src/credentials/providers](/src/credentials/providers) as they are implemented.
+At the time this document was written, Shared Credentials are the only supported Credentials. Additional credentials providers will reside at [/src/credentials/providers](/packages/toolkit/src/credentials/providers) as they are implemented.
 
 ### Shared Credentials Profiles
 
-Profiles are retrieved from the user's shared credentials files. The profile is handled and validated differently based on which fields are present. Handling and validation logic can be found in [sharedCredentialsProvider.ts](/src/credentials/providers/sharedCredentialsProvider.ts).
+Profiles are retrieved from the user's shared credentials files. The profile is handled and validated differently based on which fields are present. Handling and validation logic can be found in [sharedCredentialsProvider.ts](/packages/toolkit/src/credentials/providers/sharedCredentialsProvider.ts).
 
 Only profiles that are considered valid are provided to the toolkit. When validation issues are detected, they are written to the logs to help users understand why the toolkit is having difficulties with a profile. Users running the 'Connect to AWS' command will not see invalid profiles in the list of Credentials.
 

designs/modify-resources-attached-to-code-pipeline.md

@@ -150,7 +150,7 @@ The confirmation prompt can be implemented using one of two UI facilities:
 
 ### API Client
 
-The `ResourceGroupsTaggingAPI` service client will be used. The client will be set up in a manner consistent with the Lambda and CloudFormation clients, allowing the clients to be stubbed out in unit tests. See the Toolkit's Lambda Client [Interface](/src/shared/clients/lambdaClient.ts) and [Implementation](/src/shared/clients/defaultLambdaClient.ts) as an example. A new client factory will be added to [ToolkitClientBuilder](/src/shared/clients/toolkitClientBuilder.ts) and [DefaultToolkitClientBuilder](/src/shared/clients/defaultToolkitClientBuilder.ts).
+The `ResourceGroupsTaggingAPI` service client will be used. The client will be set up in a manner consistent with the Lambda and CloudFormation clients, allowing the clients to be stubbed out in unit tests. See the Toolkit's Lambda Client [Interface](/packages/toolkit/src/shared/clients/lambdaClient.ts) and [Implementation](/packages/toolkit/src/shared/clients/defaultLambdaClient.ts) as an example. A new client factory will be added to [ToolkitClientBuilder](/packages/toolkit/src/shared/clients/toolkitClientBuilder.ts) and [DefaultToolkitClientBuilder](/packages/toolkit/src/shared/clients/defaultToolkitClientBuilder.ts).
 
 ### <a id="fallback-handling"></a>Fallback Handling
 

docs/ARCHITECTURE.md

@@ -2,11 +2,42 @@
 
 An overview of the architecture for various components within the Toolkit.
 
+## Monorepo Structure
+
+This project is currently set up as a typescript monorepo with a single subproject.
+We are currently working on splitting the Toolkit into various subprojects to help with
+sharing code between modules, browser extension development, etc. For now, there is just
+one monolithic subproject with all the extension functionality: [`packages/toolkit/`](./packages/toolkit/).
+
+Unless otherwise stated, the documentation throughout this project is referring to the code and
+functionality of that subproject.
+
+Current quirks of the current monorepo status that should be resolved/evaluated in later versions (TODO):
+
+-   The [root package.json](../package.json) contains common dependencies for subprojects, and workspace
+    entries for each of the subprojects.
+    -   This package contains shortcuts to some of the `npm` scripts found in the subproject(s).
+    -   Other scripts, like `createRelease` and `newChange` run at the root level.
+    -   To run a script not present in the root `package.json`, use `npm run -w packages/toolkit <script>`
+-   `coverage/`, `.test-reports/`, `node_modules/` are hoisted to the project root. As more subprojects are added,
+    we will need to evaluate how to merge and publish coverage reports.
+    -   `dist/` however remains at the subproject level, along with a local `node_modules/`. See [`npm workspaces`](https://docs.npmjs.com/cli/v8/using-npm/workspaces)
+        for more info on how `node_modules/` hoisting works.
+    -   Because of `node_modules/` hoisting, references to this folder in code access the root project modules folder. This may be
+        an issue if more subprojects are added and the contents of the root and local modules folders differ.
+-   [`globalSetup.test.ts`](../packages/toolkit/src/test/globalSetup.test.ts) should be configured to work as a library/run tests for all subprojects.
+-   Subproject `tsconfig.json`s should extend a root `tsconfig.packages.json`.
+-   Linting tests should run at the root level, not subproject level.
+-   `packages/toolkit/scripts/` should be generalized and moved to the root of the project as needed.
+-   LICENSE, README.md, and other non-code artifacts that must be packaged into the .vsix are currently
+    being copied into the packaging subproject directory from the root project directory as part of the `copyFiles` task.
+-   [**Running the test suites in VSCode has changed**](../CONTRIBUTING.md#test)
+
 ## Commands
 
 Many parts of the VS Code API relies on the use of 'commands' which are simply functions bound to a global ID. For small projects, this simplicity is great. But the base API doesn't offer a lot of common functionality that a large project might want: logging, error handling, etc.
 
-For the Toolkit, common command functionality is implemented in [Commands](../src/shared/vscode/commands2.ts). The goal with this abstraction is to increase consistency across the Toolkit for anything related to commands.
+For the Toolkit, common command functionality is implemented in [Commands](../packages/toolkit/src/shared/vscode/commands2.ts). The goal with this abstraction is to increase consistency across the Toolkit for anything related to commands.
 
 ### Examples
 
@@ -90,7 +121,7 @@ _See also [CODE_GUIDELINES.md](./CODE_GUIDELINES.md#exceptions)._
 
 Large applications often have a correspondingly large number of failure points. For feature-level logic, these failures are often non-recoverable. The best we can do is show the user that something went wrong and maybe offer guidance on how to fix it.
 
-Because this is such a common pattern, shared error handling logic is defined by `ToolkitError` found [here](../src/shared/errors.ts). This class provides the basic structure for errors throughout the Toolkit.
+Because this is such a common pattern, shared error handling logic is defined by `ToolkitError` found [here](../packages/toolkit/src/shared/errors.ts). This class provides the basic structure for errors throughout the Toolkit.
 
 ### Chaining
 
@@ -100,7 +131,7 @@ By adding additional information as the exception bubbles up, we can create a be
 
 ### Handlers
 
-Any code paths exercised via `Commands` will have errors handled by `handleError` in [extensions.ts](../src/extension.ts). A better API for error handling across more than just commands will be added in a future PR.
+Any code paths exercised via `Commands` will have errors handled by `handleError` in [extensions.ts](../packages/toolkit/src/extension.ts). A better API for error handling across more than just commands will be added in a future PR.
 
 ### Best Practices
 

docs/CODE_GUIDELINES.md

@@ -183,7 +183,7 @@ _See also [ARCHITECTURE.md](./ARCHITECTURE.md#exceptions)._
 
     -   [Testing Refresh Button](https://github.com/aws/aws-toolkit-vscode/blob/b34c8f7650c862c388992781844695b014b5d974/src/test/shared/ui/prompters/rolePrompter.test.ts#L58-L65)
 
--   Use [`testCommand`](../src/test/shared/vscode/testUtils.ts) for testing commands created by `Commands.declare`
+-   Use [`testCommand`](../packages/toolkit/src/test/shared/vscode/testUtils.ts) for testing commands created by `Commands.declare`
     -   Prefer executing the real command directly when possible
 
 ## Code guidelines