diff --git a/CHANGELOG.md b/CHANGELOG.md index 897aa2742..dc7f6ea8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Git ### Added +- Added support for busybox sh +- Added flag --rcfile to specify an rc file by name. +- Added `extended-analysis=true` directive to enable/disable dataflow analysis + (with a corresponding --extended-analysis flag). - SC2324: Warn when x+=1 appends instead of increments - SC2325: Warn about multiple `!`s in dash/sh. - SC2326: Warn about `foo | ! bar` in bash/dash/sh. @@ -9,7 +13,6 @@ - SC3015: Warn bashism `test _ =~ _` like in [ ] - SC3016: Warn bashism `test -v _` like in [ ] - SC3017: Warn bashism `test -a _` like in [ ] -- Added support for busybox sh ### Fixed - source statements with here docs now work correctly diff --git a/shellcheck.1.md b/shellcheck.1.md index 42a0429b7..b2bef3c31 100644 --- a/shellcheck.1.md +++ b/shellcheck.1.md @@ -56,6 +56,13 @@ not warn at all, as `ksh` supports decimals in arithmetic contexts. options are cumulative, but all the codes can be specified at once, comma-separated as a single argument. +**--extended-analysis=true/false** + +: Enable/disable Dataflow Analysis to identify more issues (default true). If + ShellCheck uses too much CPU/RAM when checking scripts with several + thousand lines of code, extended analysis can be disabled with this flag + or a directive. This flag overrides directives and rc files. + **-f** *FORMAT*, **--format=***FORMAT* : Specify the output format of shellcheck, which prints its results in the @@ -249,6 +256,12 @@ Valid keys are: : Enable an optional check by name, as listed with **--list-optional**. Only file-wide `enable` directives are considered. +**extended-analysis** +: Set to true/false to enable/disable dataflow analysis. Specifying + `# shellcheck extended-analysis=false` in particularly large (2000+ line) + auto-generated scripts will reduce ShellCheck's resource usage at the + expense of certain checks. Extended analysis is enabled by default. + **external-sources** : Set to `true` in `.shellcheckrc` to always allow ShellCheck to open arbitrary files from 'source' statements (the way most tools do). diff --git a/shellcheck.hs b/shellcheck.hs index e933d6c78..def365465 100644 --- a/shellcheck.hs +++ b/shellcheck.hs @@ -102,6 +102,8 @@ options = [ (ReqArg (Flag "include") "CODE1,CODE2..") "Consider only given types of warnings", Option "e" ["exclude"] (ReqArg (Flag "exclude") "CODE1,CODE2..") "Exclude types of warnings", + Option "" ["extended-analysis"] + (ReqArg (Flag "extended-analysis") "bool") "Perform dataflow analysis (default true)", Option "f" ["format"] (ReqArg (Flag "format") "FORMAT") $ "Output format (" ++ formatList ++ ")", @@ -384,6 +386,14 @@ parseOption flag options = } } + Flag "extended-analysis" str -> do + value <- parseBool str + return options { + checkSpec = (checkSpec options) { + csExtendedAnalysis = Just value + } + } + -- This flag is handled specially in 'process' Flag "format" _ -> return options @@ -401,6 +411,14 @@ parseOption flag options = throwError SyntaxFailure return (Prelude.read num :: Integer) + parseBool str = do + case str of + "true" -> return True + "false" -> return False + _ -> do + printErr $ "Invalid boolean, expected true/false: " ++ str + throwError SyntaxFailure + ioInterface :: Options -> [FilePath] -> IO (SystemInterface IO) ioInterface options files = do inputs <- mapM normalize files diff --git a/src/ShellCheck/AST.hs b/src/ShellCheck/AST.hs index 5c20416e3..ca05c9868 100644 --- a/src/ShellCheck/AST.hs +++ b/src/ShellCheck/AST.hs @@ -152,6 +152,7 @@ data Annotation = | ShellOverride String | SourcePath String | ExternalSources Bool + | ExtendedAnalysis Bool deriving (Show, Eq) data ConditionType = DoubleBracket | SingleBracket deriving (Show, Eq) diff --git a/src/ShellCheck/ASTLib.hs b/src/ShellCheck/ASTLib.hs index 88089ea22..6b26b220b 100644 --- a/src/ShellCheck/ASTLib.hs +++ b/src/ShellCheck/ASTLib.hs @@ -910,5 +910,11 @@ getEnableDirectives root = T_Annotation _ list _ -> [s | EnableComment s <- list] _ -> [] +getExtendedAnalysisDirective :: Token -> Maybe Bool +getExtendedAnalysisDirective root = + case root of + T_Annotation _ list _ -> listToMaybe $ [s | ExtendedAnalysis s <- list] + _ -> Nothing + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Analytics.hs b/src/ShellCheck/Analytics.hs index 3af0455db..f88584202 100644 --- a/src/ShellCheck/Analytics.hs +++ b/src/ShellCheck/Analytics.hs @@ -1262,7 +1262,8 @@ checkNumberComparisons params (TC_Binary id typ op lhs rhs) = do str = concat $ oversimplify c var = getBracedReference str in fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup var $ CF.variablesInScope state return $ CF.numericalStatus (CF.variableValue value) >= CF.NumericalStatusMaybe _ -> @@ -2143,7 +2144,8 @@ checkSpacefulnessCfg' dirtyPass params token@(T_DollarBraced id _ list) = && not (usedAsCommandName parents token) isClean = fromMaybe False $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id value <- Map.lookup name $ CF.variablesInScope state return $ isCleanState value @@ -4896,7 +4898,8 @@ prop_checkCommandIsUnreachable3 = verifyNot checkCommandIsUnreachable "foo; bar checkCommandIsUnreachable params t = case t of T_Pipeline {} -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard . not $ CF.stateIsReachable state guard . not $ isSourced params t return $ info id 2317 "Command appears to be unreachable. Check usage (or ignore if invoked indirectly)." @@ -4918,14 +4921,15 @@ checkOverwrittenExitCode params t = _ -> return () where check id = sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id let exitCodeIds = CF.exitCodes state guard . not $ S.null exitCodeIds let idToToken = idMap params exitCodeTokens <- traverse (\k -> Map.lookup k idToToken) $ S.toList exitCodeIds return $ do - when (all isCondition exitCodeTokens && not (usedUnconditionally t exitCodeIds)) $ + when (all isCondition exitCodeTokens && not (usedUnconditionally cfga t exitCodeIds)) $ warn id 2319 "This $? refers to a condition, not a command. Assign to a variable to avoid it being overwritten." when (all isPrinting exitCodeTokens) $ warn id 2320 "This $? refers to echo/printf, not a previous command. Assign to variable to avoid it being overwritten." @@ -4938,8 +4942,8 @@ checkOverwrittenExitCode params t = -- If we don't do anything based on the condition, assume we wanted the condition itself -- This helps differentiate `x; [ $? -gt 0 ] && exit $?` vs `[ cond ]; exit $?` - usedUnconditionally t testIds = - all (\c -> CF.doesPostDominate (cfgAnalysis params) (getId t) c) testIds + usedUnconditionally cfga t testIds = + all (\c -> CF.doesPostDominate cfga (getId t) c) testIds isPrinting t = case getCommandBasename t of @@ -5009,7 +5013,8 @@ prop_checkPlusEqualsNumber9 = verifyNot checkPlusEqualsNumber "declare -ia var; checkPlusEqualsNumber params t = case t of T_Assignment id Append var _ word -> sequence_ $ do - state <- CF.getIncomingState (cfgAnalysis params) id + cfga <- cfgAnalysis params + state <- CF.getIncomingState cfga id guard $ isNumber state word guard . not $ fromMaybe False $ CF.variableMayBeDeclaredInteger state var return $ warn id 2324 "var+=1 will append, not increment. Use (( var += 1 )), declare -i var, or quote number to silence." diff --git a/src/ShellCheck/AnalyzerLib.hs b/src/ShellCheck/AnalyzerLib.hs index 944b12d88..d265ace69 100644 --- a/src/ShellCheck/AnalyzerLib.hs +++ b/src/ShellCheck/AnalyzerLib.hs @@ -104,7 +104,7 @@ data Parameters = Parameters { -- map from token id to start and end position tokenPositions :: Map.Map Id (Position, Position), -- Result from Control Flow Graph analysis (including data flow analysis) - cfgAnalysis :: CF.CFGAnalysis + cfgAnalysis :: Maybe CF.CFGAnalysis } deriving (Show) -- TODO: Cache results of common AST ops here @@ -197,8 +197,10 @@ makeCommentWithFix severity id code str fix = } in force withFix +-- makeParameters :: CheckSpec -> Parameters makeParameters spec = params where + extendedAnalysis = fromMaybe True $ msum [asExtendedAnalysis spec, getExtendedAnalysisDirective root] params = Parameters { rootNode = root, shellType = fromMaybe (determineShell (asFallbackShell spec) root) $ asShellType spec, @@ -229,7 +231,9 @@ makeParameters spec = params parentMap = getParentTree root, variableFlow = getVariableFlow params root, tokenPositions = asTokenPositions spec, - cfgAnalysis = CF.analyzeControlFlow cfParams root + cfgAnalysis = do + guard extendedAnalysis + return $ CF.analyzeControlFlow cfParams root } cfParams = CF.CFGParameters { CF.cfLastpipe = hasLastpipe params, diff --git a/src/ShellCheck/Checker.hs b/src/ShellCheck/Checker.hs index 6c9166fc5..0cfc3ab14 100644 --- a/src/ShellCheck/Checker.hs +++ b/src/ShellCheck/Checker.hs @@ -25,6 +25,7 @@ import ShellCheck.ASTLib import ShellCheck.Interface import ShellCheck.Parser +import Debug.Trace -- DO NOT SUBMIT import Data.Either import Data.Functor import Data.List @@ -86,6 +87,7 @@ checkScript sys spec = do asCheckSourced = csCheckSourced spec, asExecutionMode = Executed, asTokenPositions = tokenPositions, + asExtendedAnalysis = csExtendedAnalysis spec, asOptionalChecks = getEnableDirectives root ++ csOptionalChecks spec } where as = newAnalysisSpec root let analysisMessages = @@ -520,5 +522,43 @@ prop_hereDocsWillHaveParsedIndices = null result where result = check "#!/bin/bash\nmy_array=(a b)\ncat <> ./test\n $(( 1 + my_array[1] ))\nEOF" +prop_rcCanSuppressDfa = null result + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\nexit; foo;" + } + +prop_fileCanSuppressDfa = null $ traceShowId result + where + result = checkWithRc "" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa1 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;" + } + +prop_fileWinsWhenSuppressingDfa2 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;" + } + +prop_flagWinsWhenSuppressingDfa1 = result == [2317] + where + result = checkWithRc "extended-analysis=false" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=false\nexit; foo;", + csExtendedAnalysis = Just True + } + +prop_flagWinsWhenSuppressingDfa2 = null result + where + result = checkWithRc "extended-analysis=true" emptyCheckSpec { + csScript = "#!/bin/sh\n# shellcheck extended-analysis=true\nexit; foo;", + csExtendedAnalysis = Just False + } + return [] runTests = $quickCheckAll diff --git a/src/ShellCheck/Checks/Commands.hs b/src/ShellCheck/Checks/Commands.hs index 97c9088ba..c10016eda 100644 --- a/src/ShellCheck/Checks/Commands.hs +++ b/src/ShellCheck/Checks/Commands.hs @@ -1430,26 +1430,28 @@ prop_checkBackreferencingDeclaration6 = verify (checkBackreferencingDeclaration prop_checkBackreferencingDeclaration7 = verify (checkBackreferencingDeclaration "declare") "declare x=var $k=$x" checkBackreferencingDeclaration cmd = CommandCheck (Exactly cmd) check where - check t = foldM_ perArg M.empty $ arguments t + check t = do + cfga <- asks cfgAnalysis + when (isJust cfga) $ + foldM_ (perArg $ fromJust cfga) M.empty $ arguments t - perArg leftArgs t = + perArg cfga leftArgs t = case t of T_Assignment id _ name idx t -> do - warnIfBackreferencing leftArgs $ t:idx + warnIfBackreferencing cfga leftArgs $ t:idx return $ M.insert name id leftArgs t -> do - warnIfBackreferencing leftArgs [t] + warnIfBackreferencing cfga leftArgs [t] return leftArgs - warnIfBackreferencing backrefs l = do - references <- findReferences l + warnIfBackreferencing cfga backrefs l = do + references <- findReferences cfga l let reused = M.intersection backrefs references mapM msg $ M.toList reused msg (name, id) = warn id 2318 $ "This assignment is used again in this '" ++ cmd ++ "', but won't have taken effect. Use two '" ++ cmd ++ "'s." - findReferences list = do - cfga <- asks cfgAnalysis + findReferences cfga list = do let graph = CF.graph cfga let nodesMap = CF.tokenToNodes cfga let nodes = S.unions $ map (\id -> M.findWithDefault S.empty id nodesMap) $ map getId $ list diff --git a/src/ShellCheck/Checks/ControlFlow.hs b/src/ShellCheck/Checks/ControlFlow.hs index 9b7635ee6..d23fa1532 100644 --- a/src/ShellCheck/Checks/ControlFlow.hs +++ b/src/ShellCheck/Checks/ControlFlow.hs @@ -78,7 +78,7 @@ controlFlowEffectChecks = [ runNodeChecks :: [ControlFlowNodeCheck] -> ControlFlowCheck runNodeChecks perNode = do cfg <- asks cfgAnalysis - runOnAll cfg + sequence_ $ runOnAll <$> cfg where getData datas n@(node, label) = do (pre, post) <- M.lookup node datas diff --git a/src/ShellCheck/Interface.hs b/src/ShellCheck/Interface.hs index c574cee14..04e3c5aef 100644 --- a/src/ShellCheck/Interface.hs +++ b/src/ShellCheck/Interface.hs @@ -21,11 +21,11 @@ module ShellCheck.Interface ( SystemInterface(..) - , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csOptionalChecks) + , CheckSpec(csFilename, csScript, csCheckSourced, csIncludedWarnings, csExcludedWarnings, csShellTypeOverride, csMinSeverity, csIgnoreRC, csExtendedAnalysis, csOptionalChecks) , CheckResult(crFilename, crComments) , ParseSpec(psFilename, psScript, psCheckSourced, psIgnoreRC, psShellTypeOverride) , ParseResult(prComments, prTokenPositions, prRoot) - , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asOptionalChecks) + , AnalysisSpec(asScript, asShellType, asFallbackShell, asExecutionMode, asCheckSourced, asTokenPositions, asExtendedAnalysis, asOptionalChecks) , AnalysisResult(arComments) , FormatterOptions(foColorOption, foWikiLinkCount) , Shell(Ksh, Sh, Bash, Dash, BusyboxSh) @@ -100,6 +100,7 @@ data CheckSpec = CheckSpec { csIncludedWarnings :: Maybe [Integer], csShellTypeOverride :: Maybe Shell, csMinSeverity :: Severity, + csExtendedAnalysis :: Maybe Bool, csOptionalChecks :: [String] } deriving (Show, Eq) @@ -124,6 +125,7 @@ emptyCheckSpec = CheckSpec { csIncludedWarnings = Nothing, csShellTypeOverride = Nothing, csMinSeverity = StyleC, + csExtendedAnalysis = Nothing, csOptionalChecks = [] } @@ -174,6 +176,7 @@ data AnalysisSpec = AnalysisSpec { asExecutionMode :: ExecutionMode, asCheckSourced :: Bool, asOptionalChecks :: [String], + asExtendedAnalysis :: Maybe Bool, asTokenPositions :: Map.Map Id (Position, Position) } @@ -184,6 +187,7 @@ newAnalysisSpec token = AnalysisSpec { asExecutionMode = Executed, asCheckSourced = False, asOptionalChecks = [], + asExtendedAnalysis = Nothing, asTokenPositions = Map.empty } diff --git a/src/ShellCheck/Parser.hs b/src/ShellCheck/Parser.hs index 130d956fa..9cc5e02c8 100644 --- a/src/ShellCheck/Parser.hs +++ b/src/ShellCheck/Parser.hs @@ -1058,6 +1058,16 @@ readAnnotationWithoutPrefix sandboxed = do "This shell type is unknown. Use e.g. sh or bash." return [ShellOverride shell] + "extended-analysis" -> do + pos <- getPosition + value <- plainOrQuoted $ many1 letter + case value of + "true" -> return [ExtendedAnalysis True] + "false" -> return [ExtendedAnalysis False] + _ -> do + parseNoteAt pos ErrorC 1146 "Unknown extended-analysis value. Expected true/false." + return [] + "external-sources" -> do pos <- getPosition value <- plainOrQuoted $ many1 letter