Enhancements/big_project_mode

extension/client/src/linter.ts

@@ -78,7 +78,7 @@ export function initialise(context: ExtensionContext) {
 
 						case `streamfile`:
 							const config = instance.getConfig();
-							if (config.homeDirectory) {
+							if (config && config.homeDirectory) {
 								configPath = path.posix.join(config.homeDirectory, `.vscode`, `rpglint.json`)
 							}
 							break;

extension/client/src/requests.ts

@@ -100,16 +100,20 @@ export default function buildRequestHandlers(client: LanguageClient) {
 
 		const instance = getInstance();
 
-		const content = instance?.getContent();
-		const config = instance?.getConfig()!;
+		if (instance) {
+			const content = instance?.getContent();
+			const config = instance?.getConfig()!;
 
-		if (includePaths.length === 0) {
-			includePaths.push(config.homeDirectory);
-		}
+			if (content && config) {
+				if (includePaths.length === 0) {
+					includePaths.push(config.homeDirectory);
+				}
 
-		const resolvedPath = await content?.streamfileResolve(bases, includePaths);
+				const resolvedPath = await content?.streamfileResolve(bases, includePaths);
 
-		return resolvedPath;
+				return resolvedPath;
+			}
+		}
  });
 
 	/**

extension/server/src/providers/definition.ts

@@ -14,7 +14,7 @@ export default async function definitionProvider(handler: DefinitionParams): Pro
 			const possibleInclude = Parser.getIncludeFromDirective(editingLine);
 
 			if (possibleInclude) {
-				const include = await parser.includeFileFetch(currentPath, possibleInclude);
+				const include = await parser.includeFileFetch(currentPath, possibleInclude.content);
 				if (include.found && include.uri) {
 					return Location.create(include.uri, Range.create(0, 0, 0, 0));
 				}

extension/server/src/providers/hover.ts

@@ -95,8 +95,8 @@ export default async function hoverProvider(params: HoverParams): Promise<Hover|
 					const includeDirective = Parser.getIncludeFromDirective(lineContent);
 
 					if (includeDirective) {
-						const include = await parser.includeFileFetch(currentPath, includeDirective);
-						let displayName = includeDirective;
+						const include = await parser.includeFileFetch(currentPath, includeDirective.content);
+						let displayName = includeDirective.content;
 
 						if (include.found && include.uri) {
 							const foundUri = URI.parse(include.uri);

extension/server/src/providers/project/index.ts

@@ -9,7 +9,12 @@ import { URI } from 'vscode-uri';
 import { glob } from "glob";
 import * as path from "path";
 import { TextDocument } from 'vscode-languageserver-textdocument';
-const projectFilesGlob = `**/*.{rpgle,sqlrpgle,rpgleinc}`;
+const projectFilesGlob = `**/*.{rpgle,sqlrpgle,rpgleinc,rpgleh}`;
+
+interface iProject {
+	big?: boolean;
+	includePath?: string[]
+}
 
 export let includePath: {[workspaceUri: string]: string[]} = {};
 
@@ -58,21 +63,28 @@ export async function initialise() {
 }
 
 async function loadWorkspace() {
+	const progress = await connection.window.createWorkDoneProgress();
+
 	const workspaces = await connection.workspace.getWorkspaceFolders();
+	let handleBigProjects = false;
+	progress.begin(`RPGLE`, undefined, `Loading workspaces`);
 
 	if (workspaces) {
 		let uris: string[] = [];
 
-		workspaces.forEach((workspaceUri => {
+		for (const workspaceUri of workspaces) {
+
 			const folderPath = URI.parse(workspaceUri.uri).fsPath;
 
+			progress.report(`Starting search of ${workspaceUri.name}`);
 			console.log(`Starting search of: ${folderPath}`);
 			const files = glob.sync(projectFilesGlob, {
 				cwd: folderPath,
 				absolute: true,
 				nocase: true,
 			});
 
+			progress.report(`Found RPGLE files: ${files.length}`);
 			console.log(`Found RPGLE files: ${files.length}`);
 
 			uris.push(...files.map(file => URI.from({
@@ -93,30 +105,45 @@ async function loadWorkspace() {
 					path: base
 				}).toString();
 
-				updateIProj(iprojUri);
-			}
-		}));
+				const iproj = await updateIProj(iprojUri);
 
-		if (uris.length < 1000) {
-			for (const uri of uris) {
-				await loadLocalFile(uri);
+				if (iproj.big) {
+					handleBigProjects = true;
+				}
 			}
+		};
+
+		if (handleBigProjects) {
+			progress.report(`Big mode detected!`);
+			console.log(`Big mode detected!`);
+		}
+
+		if (uris.length < 1000 || handleBigProjects) {
+
+			await Promise.allSettled(uris.map((uri, i) => {
+				progress.report(`Loading ${i}/${uris.length}`);
+				return loadLocalFile(uri);
+			}));
+
 		} else {
+			progress.report(`Disabling project mode for large project.`);
 			console.log(`Disabling project mode for large project.`);
 			isEnabled = false;
 		}
 	}
+
+	progress.done();
 }
 
-async function updateIProj(uri: string) {
+async function updateIProj(uri: string): Promise<iProject> {
 	const workspace = await getWorkspaceFolder(uri);
 	if (workspace) {
 		const document = await getTextDoc(uri);
 		const content = document?.getText();
 
 		if (content) {
 			try {
-				const asJson = JSON.parse(content);
+				const asJson = JSON.parse(content) as iProject;
 				if (asJson.includePath && Array.isArray(asJson.includePath)) {
 					const includeArray: any[] = asJson.includePath;
 
@@ -129,19 +156,23 @@ async function updateIProj(uri: string) {
 					}
 				}
 
+				return asJson;
+
 			} catch (e) {
 				console.log(`Unable to parse JSON in ${uri}.`);
 			}
 		}
 	}
+
+	return {};
 }
 
 async function loadLocalFile(uri: string) {
 	const document = await getTextDoc(uri);
 
 	if (document) {
 		const content = document?.getText();
-		const cache = await parser.getDocs(uri, content);
+		const cache = await parser.getDocs(uri, content, {withIncludes: true, butIgnoreMembers: true});
 		if (cache) {
 			if (content.length >= 6 && content.substring(0, 6).toUpperCase() === `**FREE`) {
 				Linter.getErrors({

extension/server/src/server.ts

@@ -29,6 +29,7 @@ import implementationProvider from './providers/implementation';
 import { dspffdToRecordFormats, parseMemberUri } from './data';
 import path = require('path');
 import { existsSync } from 'fs';
+import { readFile } from 'fs/promises';
 
 let hasConfigurationCapability = false;
 let hasWorkspaceFolderCapability = false;
@@ -124,11 +125,12 @@ let fetchingInProgress: { [fetchKey: string]: boolean } = {};
 parser.setIncludeFileFetch(async (stringUri: string, includeString: string) => {
 	const currentUri = URI.parse(stringUri);
 	const uriPath = currentUri.path;
+	const isLocal = ![`streamfile`, `member`].includes(currentUri.scheme);
 
 	let cleanString: string | undefined;
 	let validUri: string | undefined;
 
-	if (!fetchingInProgress[includeString]) {
+	if (fetchingInProgress[includeString] !== true || isLocal) {
 		fetchingInProgress[includeString] = true;
 
 		// Right now we are resolving based on the base file schema.
@@ -144,7 +146,7 @@ parser.setIncludeFileFetch(async (stringUri: string, includeString: string) => {
 		}
 
 		if (isUnixPath) {
-			if (![`streamfile`, `member`].includes(currentUri.scheme)) {
+			if (isLocal) {
 				// Local file system search (scheme is usually file)
 				const workspaceFolders = await connection.workspace.getWorkspaceFolders();
 				let workspaceFolder: WorkspaceFolder | undefined;
@@ -153,10 +155,6 @@ parser.setIncludeFileFetch(async (stringUri: string, includeString: string) => {
 				}
 
 				if (Project.isEnabled) {
-					// Project mode is enable. Let's do a search for the path.
-					validUri = await validateUri(cleanString, currentUri.scheme);
-
-				} else {
 					// Because project mode is disabled, likely due to the large workspace, we don't search
 					if (workspaceFolder) {
 						cleanString = path.posix.join(URI.parse(workspaceFolder.uri).path, cleanString)
@@ -170,7 +168,15 @@ parser.setIncludeFileFetch(async (stringUri: string, includeString: string) => {
 						: undefined;
 				}
 
-				if (!validUri) {
+				if (validUri) {
+					console.log(`Valid local: ${validUri}`);
+					const validSource = await readFile(cleanString, {encoding: `utf-8`});
+					return {
+						found: true,
+						uri: validUri,
+						lines: validSource.split(`\n`)
+					};
+				} else {
 					// Ok, no local file was found. Let's see if we can do a server lookup?
 					const foundStreamfile = await streamfileResolve(stringUri, [cleanString]);
 

language/parser.js

@@ -122,7 +122,7 @@ export default class Parser {
 
   /**
 	 * @param {string} line 
-	 * @returns {string|undefined}
+	 * @returns {{content: string, isMember: boolean}|undefined}
 	 */
   static getIncludeFromDirective(line) {
     if (line.includes(`*`)) return; // Likely comment
@@ -138,14 +138,21 @@ export default class Parser {
     };
 
     if (directivePosition >= 0) {
-      return line.substring(directivePosition+directiveLength).trim();
+      let includeString = line.substring(directivePosition+directiveLength).trim();
+
+      const hasQuotes = (includeString.startsWith(`'`) && includeString.endsWith(`'`)) || (includeString.startsWith(`"`) && includeString.endsWith(`"`))
+      const isUnixPath = hasQuotes || (includeString.includes(`/`) && !includeString.includes(`,`));
+      return {
+        content: includeString,
+        isMember: !isUnixPath
+      };
     }
   }
 
   /**
    * @param {string} workingUri
    * @param {string} [content] 
-   * @param {{withIncludes?: boolean, ignoreCache?: boolean}} options
+   * @param {{withIncludes?: boolean, butIgnoreMembers?: boolean, ignoreCache?: boolean}} options
    * @returns {Promise<Cache|undefined>}
    */
   async getDocs(workingUri, content, options = {withIncludes: true}) {
@@ -302,13 +309,17 @@ export default class Parser {
         const includePath = Parser.getIncludeFromDirective(line);
 
         if (includePath) {
-          const include = await this.includeFileFetch(workingUri, includePath);
-          if (include.found) {
-            files[include.uri] = include.lines;
-            scopes[0].includes.push({
-              toPath: include.uri,
-              line: i
-            });
+          const isAllowed = includePath.isMember === false || (includePath.isMember && options.butIgnoreMembers !== true);
+
+          if (isAllowed) {
+            const include = await this.includeFileFetch(workingUri, includePath.content);
+            if (include.found) {
+              files[include.uri] = include.lines;
+              scopes[0].includes.push({
+                toPath: include.uri,
+                line: i
+              });
+            }
           }
         }
       }