New Vespa Explorer and syntax highlighting

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to the "vscode-vespa" extension will be documented in this file.
 
+## [1.1.1]
+
+- Vespa Explorer
+  - Browse cluster doc-types and fields
+  - Browse and download application content
+  - Removed the connection info
+- Simple Highlighting Syntax for Vespa schemas (*.sd)
+- WIP: Added some simple coloring for the "yql" property in the json highlight syntax
+
+
 ## [1.1.0]
 
 - Add button to save YQL result json as file.

README.md

@@ -5,16 +5,21 @@ The [VS Code Vespa extension](https://github.com/pehrs/vscode-vespa) provides ri
 
 ## Features
 
-* Run YQL requests against your vespa cluster and present the result in a webview table..
+<p align="center">
+<img src="media/vscode-vespa.gif" width=75%>
+<br/>
+</p>
+
+* Run YQL requests against your vespa clusters and present the result in a webview table..
 
 * Simple completion support is given for request snippets and the YQL query string.
 
 * Optionally render a zipkin view if you enable tracing for your YQL request.
 
-<p align="center">
-<img src="media/vscode-vespa.gif" width=75%>
-<br/>
-</p>
+* Vespa Explorer 
+
+  * Explore and download application files.
+
 
 ### Keybindings
 
@@ -70,25 +75,29 @@ You can edit the config file directly in vscode by selecting the
 
 ## Known Issues
 
-### YQL Syntax
+### YQL (.yql) Syntax
+
+This is work in progress. 
 
-The syntax is just reusing the [vscode JSON syntax](https://github.com/microsoft/vscode/blob/main/extensions/json/syntaxes/JSON.tmLanguage.json) 
-until we have time to create a custom YQL syntax.
+We have started working on this by copying the [vscode JSON syntax](https://github.com/microsoft/vscode/blob/main/extensions/json/syntaxes/JSON.tmLanguage.json) to a new [yql textmate syntax file](syntaxes/yql.tmLanguage.json) and added some simple coloring for the `yql` parameter.
+
+### Schema (.sd) Syntax
+
+This is work in progress. 
+
+We have started working on this in a new [yql textmate syntax file](syntaxes/sd.tmLanguage.json)
 
 ### Zipkin traces
 The zipkin traces are work-in-progress as the Vespa trace format is quite hard to parse.
 We have use the [TransformVespaTrace.jsx](https://github.com/vespa-engine/vespa/blob/master/client/js/app/src/app/pages/querybuilder/TransformVespaTrace.jsx) 
 from [Vespa](https://github.com/vespa-engine/vespa) as the "source of truth" on how to parse the Vespa traces.
 
-### Output view
-
-The results output view will not be saved if you close vscode.
 
 ### Language Server
 
 In the source code there is a beginings of a 
 [language server](https://code.visualstudio.com/api/language-extensions/language-server-extension-guide) in the [server/](server/) dir.
-This is not enabled as the features it gives, at the moment, is limited.
+This is not enabled as the features it gives, at the moment, are limited.
 
 ## Release Notes
 

client/package.json

@@ -3,7 +3,7 @@
 	"description": "VSCode part of a language server",
 	"author": "Matti Pehrs",
 	"license": "MIT",
-	"version": "1.1.0",
+	"version": "1.1.1",
 	"publisher": "pehrs.com",	
 	"engines": {
 		"vscode": "^1.75.0"

client/src/VespaConfigExplorer.ts

@@ -8,6 +8,10 @@ import { VespaAppId } from './model/VespaAppId';
 import { VespaClusterList } from './model/VespaClusterList';
 import { VespaStatusResultsPanel } from './VespaStatusResultsPanel';
 import { VespaV2Metrics } from './model/VespaMetrics';
+import { AppContent } from './model/AppContent';
+import { text } from 'stream/consumers';
+
+const appContent: AppContent = new AppContent();
 
 export class VespaConfigExplorer {
 
@@ -21,6 +25,7 @@ export class VespaConfigExplorer {
 
 		this.registerSetDefaultCommand();
 		this.registerShowClusterStatusCommand(context);
+		this.registerOpenFile();
 
 		vscode.commands.registerCommand("vscode-vespa.refreshVespaConfig", args => {
 			if (args !== undefined) {
@@ -105,6 +110,52 @@ export class VespaConfigExplorer {
 		});
 	}
 
+
+
+	private registerOpenFile() {
+		vscode.commands.registerCommand("vscode-vespa.openSelectedFile", args => {
+			if (args !== undefined) {
+				const element = args as { key: string; data: { filename: string, cluster: string }; };
+
+				if (element.key === "file") {
+					const fileParts = element.data.filename.split("/");
+					const fn = element.data.filename.endsWith("/") ?
+						fileParts[fileParts.length - 2] :
+						fileParts[fileParts.length - 1];
+
+					const clusterName = element.data.cluster;
+					const configEndpoint = vespaConfig.getCluster(clusterName).configEndpoint;
+					appContent.fetchAppFile(configEndpoint, element.data.filename)
+						.then(textData => {
+							const fnPath = path.join(vscode.workspace.workspaceFolders[0].uri.path, fn);
+							const setting: vscode.Uri = vscode.Uri.parse(`untitled:${fnPath}`);
+							vscode.workspace.openTextDocument(setting).then((a: vscode.TextDocument) => {
+								vscode.window.showTextDocument(a, 1, false).then(e => {
+									e.edit(edit => {
+										edit.insert(new vscode.Position(0, 0), textData);										
+									});
+								});
+							}, (error: any) => {
+								showError(JSON.stringify(error));
+							});
+
+							// 
+							// FIXME: Open "new" file (Will not have a proper filename!!!)
+							// 
+							// const fnParts = element.data.filename.split(".");
+							// const language = fnParts[fnParts.length - 1];
+							// vscode.workspace.openTextDocument({
+							// 	content: textData,
+							// 	language: language
+							// }).then(newDocument => {
+							// 	vscode.window.showTextDocument(newDocument);
+							// });
+						});
+				}
+			}
+		});
+	}
+
 	private onActiveEditorChanged(): void {
 
 		if (vscode.window.activeTextEditor) {
@@ -185,9 +236,47 @@ export class VespaConfigProvider implements vscode.TreeDataProvider<{ key: strin
 				//{ key: `configEndpoint|${clusterName}`, data: clusterName, },
 				//{ key: `zipkinEndpoint|${clusterName}`, data: clusterName, },
 				{ key: `docTypes|${clusterName}`, data: clusterName, },
+				{ key: `appFiles`, data: clusterName, },
 			]);
 		}
 
+		if (element !== undefined && element.key === "appFiles") {
+			// 
+			const clusterName = element.data as string;
+			const configEndpoint = vespaConfig.getCluster(clusterName).configEndpoint;
+			return appContent.fetchAppDir(configEndpoint, "")
+				.then(files => {
+					return files.map(filename => {
+						return {
+							key: "file",
+							data: {
+								filename: filename,
+								cluster: clusterName,
+							}
+						};
+					});
+				});
+
+		}
+
+		if (element !== undefined && element.key === "file") {
+			const data = element.data as { filename: string, cluster: string };
+			const clusterName = data.cluster;
+			const configEndpoint = vespaConfig.getCluster(clusterName).configEndpoint;
+			return appContent.fetchAppDir(configEndpoint, data.filename)
+				.then(files => {
+					return files.map(filename => {
+						return {
+							key: "file",
+							data: {
+								filename: filename,
+								cluster: clusterName,
+							}
+						};
+					});
+				});
+		}
+
 		if (element !== undefined && element.key.startsWith("docTypes|")) {
 			const clusterName = element.key.replace("docTypes|", "");
 			outputChannel.appendLine("DOC types for: " + clusterName);
@@ -367,6 +456,44 @@ export class VespaConfigProvider implements vscode.TreeDataProvider<{ key: strin
 				contextValue: "VESPA_CLUSTER",
 			};
 		}
+
+		if (element.key === "appFiles") {
+			const clusterName = element.data as string;
+
+			return {
+				label: `application-files`,
+				iconPath: new vscode.ThemeIcon('file-directory'),
+				// tooltip: new vscode.MarkdownString(`$(zap) Tooltip for ${element.key}`, true),
+				collapsibleState: vscode.TreeItemCollapsibleState.Collapsed,
+				contextValue: "VESPA_EXPLORER_APP_FILES",
+			};
+		}
+
+		if (element.key === "file") {
+			const data = element.data as { filename: string, cluster: string };
+
+			const fileParts = data.filename.split("/");
+			const fn = data.filename.endsWith("/") ?
+				fileParts[fileParts.length - 2] :
+				fileParts[fileParts.length - 1];
+
+
+			return {
+				label: fn,
+				iconPath: data.filename.endsWith("/") ?
+					new vscode.ThemeIcon('file-directory') :
+					new vscode.ThemeIcon('file'),
+				// tooltip: new vscode.MarkdownString(`$(zap) Tooltip for ${element.key}`, true),
+				collapsibleState: data.filename.endsWith("/") ?
+					vscode.TreeItemCollapsibleState.Collapsed :
+					vscode.TreeItemCollapsibleState.None,
+				contextValue: data.filename.endsWith("/") ?
+					"VESPA_EXPLORER_APP_DIR" :
+					"VESPA_EXPLORER_APP_FILE",
+			};
+		}
+
+
 		switch (element.key) {
 			case "vespa-config":
 				return {

client/src/model/AppContent.test.ts

@@ -0,0 +1,31 @@
+import { AppContent } from './AppContent';
+import { VespaAppId } from './VespaAppId';
+import { VespaStatus } from './VespaStatus';
+
+
+describe('AppContent', function () {
+	describe('given localhost cluster', function () {
+		it('should return valid list of files', function () {
+
+			// FIXME: This needs to be in a integration test where we start a vespa cluster to test :-)
+
+			// const configEndpoint = "http://localhost:19071";
+			// const appContent = new AppContent();
+			// appContent.fetchAppDir(configEndpoint, "")
+			// 	.then(content => {
+			// 		console.log("content: ", content);
+
+			// 		content.map(f => {
+			// 			if (f.endsWith("/")) {
+			// 				appContent.fetchAppDir(configEndpoint, f)
+			// 					.then(content => {
+			// 						console.log(f + ", content: ", content);
+			// 					});
+			// 			} else {
+			// 				console.log("FILE: " + f);
+			// 			}
+			// 		});
+			// 	});
+		});
+	});
+});

client/src/model/AppContent.ts

@@ -0,0 +1,55 @@
+import { vespaConfig } from '../VespaConfig';
+import { fetchWithTimeout } from '../vespaUtils';
+import { VespaAppId } from './VespaAppId';
+import { VespaStatus } from './VespaStatus';
+
+
+
+export class AppContent {
+
+	configEndpoint: string = undefined;
+	status: VespaStatus = undefined;
+	appId: VespaAppId = undefined;
+
+	async fetchAppDir(configEndpoint: string, path: string): Promise<string[]> {
+		const timeoutMs = vespaConfig.httpTimeoutMs();
+
+		if(this.status === undefined) {
+			this.status = await VespaStatus.fetchVespaStatus(configEndpoint);
+		}
+		if(this.appId === undefined) {
+			this.appId = await VespaAppId.fetchAppId(configEndpoint);
+		}
+
+		const contentRoot = this.appId.getVespaContentURL(configEndpoint, this.status, "").toString();
+		const contentPath = `${contentRoot}/${path}`;
+
+		return fetchWithTimeout(contentPath, timeoutMs)
+			.then(urlResponse => urlResponse.json())
+			.then(files => (files as string[]).map(f => f.replace(contentRoot, "")));
+	}
+
+	async fetchAppFile(configEndpoint: string, path: string): Promise<any> {
+
+		if(path.endsWith("/")) {
+			return Promise.reject("Cannot get 'content' of a dir!");
+		}
+
+		const timeoutMs = vespaConfig.httpTimeoutMs();
+
+		if(this.status === undefined) {
+			this.status = await VespaStatus.fetchVespaStatus(configEndpoint);
+		}
+		if(this.appId === undefined) {
+			this.appId = await VespaAppId.fetchAppId(configEndpoint);
+		}
+
+		const contentRoot = this.appId.getVespaContentURL(configEndpoint, this.status, "").toString();
+		const contentPath = `${contentRoot}/${path}`;
+
+
+		// Assuming text data :-)
+		return fetchWithTimeout(contentPath, timeoutMs)
+			.then(urlResponse => urlResponse.text());
+	}
+}

client/src/model/VespaMetrics.test.ts

@@ -4,19 +4,18 @@ import { metrics2ContentNodes } from './testFixtures/metricsFixtures';
 import assert = require('assert');
 
 describe('VespaMetrics', function () {
-	describe('given a parsed services.xml from a single node cluster', function () {
-		it('should return valid VespaServiceXml object', function () {
-		});
-	});
+
+	// FIXME: Add more tests :-)
 
 	describe('given a metric v2 response', () => {
 		it('should return valid VespaV2Metrics object', function () {
 
 			const metrics = VespaV2Metrics.parse(metrics2ContentNodes);
 
-			console.log("metrics: ", JSON.stringify(metrics));			
-	
+			// console.log("metrics: ", JSON.stringify(metrics));			
+
 			// assert.equal(hostsAliases.hosts.size, 4);
+
 		});
 
 	});