Feature/syntax_checker

.vscode/launch.json

@@ -39,7 +39,6 @@
 				"type": "npm",
 				"script": "compile:tests"
 			},
-			"args": ["issue_239"],
 			"env": {
 				"INCLUDE_DIR": "${workspaceFolder}"
 			}

extension/server/src/providers/linter/index.ts

@@ -235,7 +235,7 @@ export async function refreshLinterDiagnostics(document: TextDocument, docs: Cac
 
 				const diagnostic = Diagnostic.create(
 					range,
-					Linter.getErrorText(error.type),
+					error.message || Linter.getErrorText(error.type!),
 					DiagnosticSeverity.Warning
 				);
 

language/linter.ts

@@ -7,6 +7,7 @@ import { Range, Position } from "./models/DataPoints";
 import opcodes from "./models/opcodes";
 import Document from "./document";
 import { IssueRange, Offset, Rules, SelectBlock } from "./parserTypes";
+import { validateTokens } from "./validator";
 
 const errorText = {
   'BlankStructNamesCheck': `Struct names cannot be blank (\`*N\`).`,
@@ -39,7 +40,8 @@ const errorText = {
   'NoExtProgramVariable': `Not allowed to use variable in EXTPGM or EXTPROC.`,
   'IncludeMustBeRelative': `Path not valid. It must be relative to the project.`,
   'SQLHostVarCheck': `Also defined in scope. Should likely be host variable.`,
-  'RequireOtherBlock': `OTHER block missing from SELECT block.`
+  'RequireOtherBlock': `OTHER block missing from SELECT block.`,
+  'Validator': `Token not expected.`
 };
 
 const skipRules = {
@@ -50,7 +52,7 @@ const skipRules = {
 };
 
 export default class Linter {
-  static getErrorText(error) {
+  static getErrorText(error: keyof Rules) {
     return errorText[error];
   }
 
@@ -118,6 +120,11 @@ export default class Linter {
       lineNumber = docStatement.range.line;
       currentIndent = docStatement.indent;
 
+      if (rules.Validator) {
+        const possibleError = validateTokens(statement);
+        if (possibleError) errors.push(possibleError);
+      }
+
       if (currentIndent >= 0) {
         skipIndentCheck = false;
 

language/parserTypes.ts

@@ -56,9 +56,15 @@ export interface Rules {
   IncludeMustBeRelative?: boolean;
   SQLHostVarCheck?: boolean;
   RequireOtherBlock?: boolean;
+  Validator?: boolean;
 
+  /** Not exposed rules below */
+
   /** When true, will update Cache will references found in linter */
   CollectReferences?: boolean;
+
+  InvalidDeclareNumber?: boolean;
+  UnexpectedEnd?: boolean;
 }
 
 export  interface DefinitionPosition {
@@ -73,15 +79,9 @@ export interface Offset {
 
 export interface IssueRange {
   offset: Offset;
-  type?: "BlankStructNamesCheck"|"QualifiedCheck"|"PrototypeCheck"|"ForceOptionalParens"|
-  "NoOCCURS"|"NoSELECTAll"|"UselessOperationCheck"|"UppercaseConstants"|"SpecificCasing"|
-  "InvalidDeclareNumber"|"IncorrectVariableCase"|"RequiresParameter"|
-  "RequiresProcedureDescription"|"StringLiteralDupe"|"RequireBlankSpecial"|
-  "CopybookDirective"|"UppercaseDirectives"|"NoSQLJoins"|"NoGlobalsInProcedures"|
-  "NoCTDATA"|"PrettyComments"|"NoGlobalSubroutines"|"NoLocalSubroutines"|"UnexpectedEnd"|
-  "NoUnreferenced"|"NoExternalTo"|"NoExecuteImmediate"|"NoExtProgramVariable"|"IncludeMustBeRelative"|
-  "SQLHostVarCheck"|"RequireOtherBlock";
+  type?: keyof Rules;
   newValue?: string;
+  message?: string;
 }
 
 export interface SelectBlock {

language/tokens.ts

@@ -17,6 +17,8 @@ enum ReadState {
   IN_COMMENT = "Comment"
 }
 
+const numReg = new RegExp(/^\d+$/)
+
 const commonMatchers: Matcher[] = [
   {
     name: `FORMAT_STATEMEMT`,
@@ -44,6 +46,18 @@ const commonMatchers: Matcher[] = [
   },
   {
     name: `IS_NUMBER`,
+    match: [
+      {
+        type: `word`,
+        match: (word) => numReg.test(word)
+      },
+    ],
+    becomes: {
+      type: `number`
+    }
+  },
+  {
+    name: `DEC_NUMBER`,
     match: [
       { type: `number` },
       { type: `dot` },
@@ -59,7 +73,7 @@ const commonMatchers: Matcher[] = [
       { type: `asterisk` },
       {
         type: `word`, match: (word) =>
-          [`CTDATA`, `BLANK`, `BLANKS`, `ZERO`, `ZEROS`, `ON`, `OFF`, `NULL`, `ISO`, `MDY`, `DMY`, `EUR`, `YMD`, `USA`, `SECONDS`, `S`, `MINUTES`, `MN`, `HOURS`, `H`, `DAYS`, `D`, `MONTHS`, `M`, `YEARS`, `Y`, `HIVAL`, `END`, `LOVAL`, `START`, `N`, `OMIT`, `STRING`, `CWIDEN`, `CONVERT`].includes(word.toUpperCase()) || word.toUpperCase().startsWith(`IN`)
+          [`CTDATA`, `BLANK`, `BLANKS`, `ZERO`, `ZEROS`, `ON`, `OFF`, `NULL`, `ISO`, `MDY`, `DMY`, `EUR`, `YMD`, `USA`, `SECONDS`, `S`, `MINUTES`, `MN`, `HOURS`, `H`, `DAYS`, `D`, `MONTHS`, `M`, `YEARS`, `Y`, `HIVAL`, `END`, `LOVAL`, `START`, `N`, `OMIT`, `STRING`, `CWIDEN`, `CONVERT`, `KEY`, `SRCSTMT`, `NOPASS`, `YES`, `NO`, `NODEBUGIO`, `TRIM`, `VARSIZE`, `CALLER`].includes(word.toUpperCase()) || word.toUpperCase().startsWith(`IN`)
       }
     ],
     becomes: {
@@ -169,6 +183,16 @@ const commonMatchers: Matcher[] = [
       type: `word`
     }
   },
+  {
+    name: `QUOTE-STRING`,
+    match: [
+      { type: `string`, match: (word) => word === `''` },
+      { type: `string`, match: (word) => word === `''` },
+    ],
+    becomes: {
+      type: `string`
+    }
+  },
   {
     name: `CTL-OPT`,
     match: [
@@ -279,6 +303,14 @@ export function tokenise(statement) {
       switch (statement[i]) {
       // When it's the string character..
       case stringChar:
+        // Silly way RPG does escape strings...
+        const escapeStringStart = (state === ReadState.IN_STRING && statement[i+1] === stringChar);
+
+        if (escapeStringStart) {
+          currentText += (stringChar + stringChar);
+          i++;
+
+        } else 
         if (state === ReadState.IN_STRING) {
           currentText += statement[i];
           result.push({ value: currentText, type: `string`, range: { start: startsAt, end: startsAt + currentText.length, line: lineNumber } });
@@ -288,8 +320,10 @@ export function tokenise(statement) {
           currentText += statement[i];
         }
 
-        // @ts-ignore
-        state = state === ReadState.IN_STRING ? ReadState.NORMAL : ReadState.IN_STRING;
+        if (escapeStringStart !== true) {
+          // @ts-ignore
+          state = state === ReadState.IN_STRING ? ReadState.NORMAL : ReadState.IN_STRING;
+        }
         break;
 
       // When it's any other character...

language/validator.ts

@@ -0,0 +1,139 @@
+import { IssueRange } from "./parserTypes";
+import { Token } from "./types";
+
+interface GenericTokenFormat {
+  not?: boolean;
+  type: string[];
+  value?: string;
+};
+
+interface GenericWhenRule {
+  when: GenericTokenFormat;
+  start?: boolean;
+  then?: GenericTokenFormat[];
+  end?: boolean;
+}
+
+const Operators = [`plus`, `minus`, `divide`, `asterisk`, `comma`, `dot`];
+const ValueTypes = [`string`, `number`, `special`, `hex`, `builtin`];
+const ExprParts = [...Operators, ...ValueTypes];
+
+const GenericRules: GenericWhenRule[] = [
+  {
+    when: {
+      type: [`format`]
+    },
+    start: true,
+    then: [],
+    end: true,
+  },
+  {
+    when: {
+      type: [`directive`, `declare`],
+    },
+    start: true
+  },
+  {
+    when: {
+      type: [`seperator`],
+    },
+    then: [{not: true, type: Operators}]
+  },
+  {
+    when: {
+      type: [`openbracket`],
+    },
+    then: [{not: true, type: Operators}]
+  },
+  {
+    when: {
+      type: [`closebracket`],
+    },
+    then: [{not: true, type: ValueTypes}]
+  },
+  {
+    when: {
+      type: Operators
+    },
+    then: [{not: true, type: Operators}]
+  },
+  {
+    when: {
+      type: ValueTypes
+    },
+    then: [{not: true, type: ValueTypes}]
+  },
+  {
+    when: {
+      type: Operators
+    },
+    then: [{type: ValueTypes}]
+  },
+];
+Object.freeze(GenericRules);
+
+export function validateTokens(tokens: Token[]): IssueRange|undefined {
+  for (let i = 0; i < tokens.length; i++) {
+    let token = tokens[i];
+    let cToken = tokens[i];
+
+    const currentRule = GenericRules.find(rule => rule.when.type.includes(cToken.type) && rule.when.value === undefined || rule.when.value === cToken.value);
+
+    if (currentRule) {
+      let cI = i;
+      if (cI !== 0 && currentRule.start) {
+        // Throw error. This can only be at the start
+        return {
+          offset: {position: cToken.range.start, end: cToken.range.end},
+          type: `Validator`,
+          message: `Token is expected at the start of statements`
+        }
+      }
+
+      if (currentRule.then && currentRule.then.length > 0) {
+        for (const thenItem of currentRule.then) {
+          cI++;
+
+          cToken = tokens[cI];
+
+          if (cToken) {
+            const typeMatch = thenItem.type.includes(cToken.type) && (thenItem.value ? thenItem.value === cToken.value : true);
+            const isError = (thenItem.not ? typeMatch : !typeMatch);
+
+            if (isError) {
+              // Token unexpected
+              return {
+                offset: {position: cToken.range.start, end: cToken.range.end},
+                type: `Validator`,
+                message: `Token not expected`
+              }
+            }
+          } else if (thenItem.not !== true) {
+            return {
+              offset: {position: token.range.start, end: token.range.end},
+              type: `Validator`,
+              message: `'${thenItem.type.join()}' is ${thenItem.not ? `not` : ``} expected.`
+            }
+          }
+        }
+
+        cI++;
+      }
+
+      if (cI !== (tokens.length-1) && currentRule.end) {
+        cToken = tokens[cI];
+
+        if (cToken) {
+        // Throw error. This can only be at the end
+          return {
+            offset: {position: cToken.range.start, end: cToken.range.end},
+            type: `Validator`,
+            message: `Token should be at the end of the statement only.`
+          }
+        }
+      }
+    }
+  }
+
+  return;
+}

schemas/rpglint.json

@@ -175,6 +175,11 @@
 			"$id": "#/properties/RequireOtherBlock",
 			"type": "boolean",
 			"description": "Require SELECT blocks to have an OTHER block."
+		},
+		"Validator": {
+			"$id": "#/properties/Validator",
+			"type": "boolean",
+			"description": "Will add additional validation against RPGLE syntax. Useful for finding common code issue."
 		}
 	},
 	"additionalProperties": true

tests/suite/basics.js

@@ -435,6 +435,7 @@ exports.indicators1 = async () => {
   const cache = await parser.getDocs(uri, lines);
 
   Linter.getErrors({ uri, content: lines }, {
+    Validator: true,
     CollectReferences: true,
   }, cache);