diff --git a/python/ql/lib/change-notes/2026-01-02-prompt-injection.md b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md new file mode 100644 index 000000000000..21f04216ecbc --- /dev/null +++ b/python/ql/lib/change-notes/2026-01-02-prompt-injection.md @@ -0,0 +1,5 @@ +--- +category: minorAnalysis +--- +* Added experimental query `py/prompt-injection` to detect potential prompt injection vulnerabilities in code using LLMs. +* Added taint flow model and type model for `agents` and `openai` modules. \ No newline at end of file diff --git a/python/ql/lib/semmle/python/Concepts.qll b/python/ql/lib/semmle/python/Concepts.qll index 0ca8a4dbef01..73d3b0d1e80f 100644 --- a/python/ql/lib/semmle/python/Concepts.qll +++ b/python/ql/lib/semmle/python/Concepts.qll @@ -325,6 +325,31 @@ private class EncodingAdditionalTaintStep extends TaintTracking::AdditionalTaint } } +/** + * A data-flow node that prompts an AI model. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `AIPrompt::Range` instead. + */ +class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range { + /** Gets an input that is used as AI prompt. */ + DataFlow::Node getAPrompt() { result = super.getAPrompt() } +} + +/** Provides a class for modeling new AI prompting mechanisms. */ +module AIPrompt { + /** + * A data-flow node that prompts an AI model. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `AIPrompt` instead. + */ + abstract class Range extends DataFlow::Node { + /** Gets an input that is used as AI prompt. */ + abstract DataFlow::Node getAPrompt(); + } +} + /** * A data-flow node that logs data. * diff --git a/python/ql/lib/semmle/python/Frameworks.qll b/python/ql/lib/semmle/python/Frameworks.qll index 7694419b41d5..f28686bf2fae 100644 --- a/python/ql/lib/semmle/python/Frameworks.qll +++ b/python/ql/lib/semmle/python/Frameworks.qll @@ -54,6 +54,7 @@ private import semmle.python.frameworks.Multidict private import semmle.python.frameworks.Mysql private import semmle.python.frameworks.MySQLdb private import semmle.python.frameworks.Numpy +private import semmle.python.frameworks.OpenAI private import semmle.python.frameworks.Opml private import semmle.python.frameworks.Oracledb private import semmle.python.frameworks.Pandas diff --git a/python/ql/lib/semmle/python/frameworks/OpenAI.qll b/python/ql/lib/semmle/python/frameworks/OpenAI.qll new file mode 100644 index 000000000000..7cd11ebabefe --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/OpenAI.qll @@ -0,0 +1,85 @@ +/** + * Provides classes modeling security-relevant aspects of the `openAI` Agents SDK package. + * See https://github.com/openai/openai-agents-python. + * As well as the regular openai python interface. + * See https://github.com/openai/openai-python. + */ + +private import python +private import semmle.python.ApiGraphs + +/** + * Provides models for agents SDK (instances of the `agents.Runner` class etc). + * + * See https://github.com/openai/openai-agents-python. + */ +module AgentSDK { + /** Gets a reference to the `agents.Runner` class. */ + API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") } + + /** Gets a reference to the `run` members. */ + API::Node runMembers() { result = classRef().getMember(["run", "run_sync", "run_streamed"]) } + + /** Gets a reference to a potential property of `agents.Runner` called input which can refer to a system prompt depending on the role specified. */ + API::Node getContentNode() { + result = runMembers().getKeywordParameter("input").getASubscript().getSubscript("content") + or + result = runMembers().getParameter(_).getASubscript().getSubscript("content") + } +} + +/** + * Provides models for Agent (instances of the `openai.OpenAI` class). + * + * See https://github.com/openai/openai-python. + */ +module OpenAI { + /** Gets a reference to the `openai.OpenAI` class. */ + API::Node classRef() { + result = + API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]).getReturn() + } + + /** Gets a reference to a potential property of `openai.OpenAI` called instructions which refers to the system prompt. */ + API::Node getContentNode() { + exists(API::Node content | + content = + classRef() + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) or + content = + classRef() + .getMember("responses") + .getMember("create") + .getKeywordParameter(["input", "instructions"]) + .getASubscript() + .getSubscript("content") or + content = + classRef() + .getMember("realtime") + .getMember("connect") + .getReturn() + .getMember("conversation") + .getMember("item") + .getMember("create") + .getKeywordParameter("item") + .getSubscript("content") or + content = + classRef() + .getMember("chat") + .getMember("completions") + .getMember("create") + .getKeywordParameter("messages") + .getASubscript() + .getSubscript("content") + | + // content + if not exists(content.getASubscript()) + then result = content + else + // content.text + result = content.getASubscript().getSubscript("text") + ) + } +} diff --git a/python/ql/lib/semmle/python/frameworks/agent.model.yml b/python/ql/lib/semmle/python/frameworks/agent.model.yml new file mode 100644 index 000000000000..5a923a335197 --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/agent.model.yml @@ -0,0 +1,6 @@ +extensions: + - addsTo: + pack: codeql/python-all + extensible: sinkModel + data: + - ['agents', 'Member[Agent].Argument[instructions:]', 'prompt-injection'] diff --git a/python/ql/lib/semmle/python/frameworks/openai.model.yml b/python/ql/lib/semmle/python/frameworks/openai.model.yml new file mode 100644 index 000000000000..245d390ab8eb --- /dev/null +++ b/python/ql/lib/semmle/python/frameworks/openai.model.yml @@ -0,0 +1,12 @@ +extensions: + - addsTo: + pack: codeql/python-all + extensible: sinkModel + data: + - ['OpenAI', 'Member[beta].Member[assistants].Member[create].Argument[instructions:]', 'prompt-injection'] + + - addsTo: + pack: codeql/python-all + extensible: typeModel + data: + - ['OpenAI', 'openai', 'Member[OpenAI,AsyncOpenAI,AzureOpenAI].ReturnValue'] diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll new file mode 100644 index 000000000000..aaa8c05418ed --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionCustomizations.qll @@ -0,0 +1,64 @@ +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ + +import python +private import semmle.python.dataflow.new.DataFlow +private import semmle.python.Concepts +private import semmle.python.dataflow.new.RemoteFlowSources +private import semmle.python.dataflow.new.BarrierGuards +private import semmle.python.frameworks.data.ModelsAsData +private import semmle.python.frameworks.OpenAI + +/** + * Provides default sources, sinks and sanitizers for detecting + * "prompt injection" + * vulnerabilities, as well as extension points for adding your own. + */ +module PromptInjection { + /** + * A data flow source for "prompt injection" vulnerabilities. + */ + abstract class Source extends DataFlow::Node { } + + /** + * A data flow sink for "prompt injection" vulnerabilities. + */ + abstract class Sink extends DataFlow::Node { } + + /** + * A sanitizer for "prompt injection" vulnerabilities. + */ + abstract class Sanitizer extends DataFlow::Node { } + + /** + * An active threat-model source, considered as a flow source. + */ + private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { } + + /** + * A prompt to an AI model, considered as a flow sink. + */ + class AIPromptAsSink extends Sink { + AIPromptAsSink() { this = any(AIPrompt p).getAPrompt() } + } + + private class SinkFromModel extends Sink { + SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() } + } + + private class PromptContentSink extends Sink { + PromptContentSink() { + this = OpenAI::getContentNode().asSink() + or + this = AgentSDK::getContentNode().asSink() + } + } + + /** + * A comparison with a constant, considered as a sanitizer-guard. + */ + class ConstCompareAsSanitizerGuard extends Sanitizer, ConstCompareBarrier { } +} diff --git a/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll new file mode 100644 index 000000000000..5c0413726e62 --- /dev/null +++ b/python/ql/lib/semmle/python/security/dataflow/PromptInjectionQuery.qll @@ -0,0 +1,25 @@ +/** + * Provides a taint-tracking configuration for detecting "prompt injection" vulnerabilities. + * + * Note, for performance reasons: only import this file if + * `PromptInjection::Configuration` is needed, otherwise + * `PromptInjectionCustomizations` should be imported instead. + */ + +private import python +import semmle.python.dataflow.new.DataFlow +import semmle.python.dataflow.new.TaintTracking +import PromptInjectionCustomizations::PromptInjection + +private module PromptInjectionConfig implements DataFlow::ConfigSig { + predicate isSource(DataFlow::Node node) { node instanceof Source } + + predicate isSink(DataFlow::Node node) { node instanceof Sink } + + predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer } + + predicate observeDiffInformedIncrementalMode() { any() } +} + +/** Global taint-tracking for detecting "prompt injection" vulnerabilities. */ +module PromptInjectionFlow = TaintTracking::Global; diff --git a/python/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp new file mode 100644 index 000000000000..ef6b9c83ac26 --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp @@ -0,0 +1,24 @@ + + + + +

Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or +operations that were not intended.

+ + + +

Sanitize user input and also avoid using user input in developer or system level prompts.

+ + + +

In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.

+ + + + +
  • OpenAI: Guardrails.
    • +     + +     diff --git a/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql new file mode 100644 index 000000000000..3bb985264fac --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql @@ -0,0 +1,20 @@ +/** + * @name Prompt injection + * @kind path-problem + * @problem.severity error + * @security-severity 5.0 + * @precision high + * @id py/prompt-injection + * @tags security + * experimental + * external/cwe/cwe-1427 + */ + +import python +import semmle.python.security.dataflow.PromptInjectionQuery +import PromptInjectionFlow::PathGraph + +from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink +where PromptInjectionFlow::flowPath(source, sink) +select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(), + "user-provided value" diff --git a/python/ql/src/experimental/Security/CWE-1427/examples/example.py b/python/ql/src/experimental/Security/CWE-1427/examples/example.py new file mode 100644 index 000000000000..a049f727b37a --- /dev/null +++ b/python/ql/src/experimental/Security/CWE-1427/examples/example.py @@ -0,0 +1,17 @@ +from flask import Flask, request +from agents import Agent +from guardrails import GuardrailAgent + +@app.route("/parameter-route") +def get_input(): + input = request.args.get("input") + + goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured. + config=Path("guardrails_config.json"), + name="Assistant", + instructions="This prompt is customized for " + input) + + badAgent = Agent( + name="Assistant", + instructions="This prompt is customized for " + input # BAD: user input in agent instruction. + ) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected new file mode 100644 index 000000000000..bbf9e5665bab --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.expected @@ -0,0 +1,94 @@ +#select +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:33:33:33:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:33:33:33:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:53:33:53:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:53:33:53:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:67:28:67:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:67:28:67:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:71:28:71:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:71:28:71:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:84:28:84:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:84:28:84:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value | +edges +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:17:13:17:19 | ControlFlowNode for request | provenance | | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:93 | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | provenance | | +| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | agent_instructions.py:17:5:17:9 | ControlFlowNode for input | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | provenance | | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58613 | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get | +| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:33:33:33:37 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:53:33:53:37 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:67:28:67:32 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:71:28:71:32 | ControlFlowNode for query | provenance | | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:84:28:84:32 | ControlFlowNode for query | provenance | | +| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep | +| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get | +| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | | +nodes +| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input | +| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember | +| openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona | +| openai_test.py:12:15:12:21 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| openai_test.py:13:5:13:9 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request | +| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute | +| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() | +| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:33:33:33:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:53:33:53:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:67:28:67:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:71:28:71:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +| openai_test.py:84:28:84:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query | +| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr | +subpaths diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref new file mode 100644 index 000000000000..08466562ffe7 --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/PromptInjection.qlref @@ -0,0 +1,2 @@ +query: experimental/Security/CWE-1427/PromptInjection.ql +postprocess: utils/test/InlineExpectationsTestQuery.ql \ No newline at end of file diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py new file mode 100644 index 000000000000..12cebc1b5831 --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/agent_instructions.py @@ -0,0 +1,38 @@ +from agents import Agent, Runner +from flask import Flask, request # $ Source +app = Flask(__name__) + +@app.route("/parameter-route") +def get_input1(): + input = request.args.get("input") + + agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection] + + result = Runner.run_sync(agent, "This is a user message.") + print(result.final_output) + + +@app.route("/parameter-route") +def get_input2(): + input = request.args.get("input") + + agent = Agent(name="Assistant", instructions="This prompt is not customized.") + result = Runner.run_sync( + agent=agent, + input=[ + { + "role": "user", + "content": input, # $Alert[py/prompt-injection] + } + ] + ) + + result2 = Runner.run_sync( + agent, + [ + { + "role": "user", + "content": input, # $Alert[py/prompt-injection] + } + ] + ) diff --git a/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py new file mode 100644 index 000000000000..2b25609670c5 --- /dev/null +++ b/python/ql/test/experimental/query-tests/Security/CWE-1427-PromptInjection/openai_test.py @@ -0,0 +1,93 @@ +from openai import OpenAI, AsyncOpenAI, AzureOpenAI +from flask import Flask, request # $ Source +app = Flask(__name__) + +client = OpenAI() +async_client = AsyncOpenAI() +azure_client = AzureOpenAI() + + +@app.route("/openai") +async def get_input_openai(): + persona = request.args.get("persona") + query = request.args.get("query") + role = request.args.get("role") + + response1 = client.responses.create( + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] + input=query, # $ Alert[py/prompt-injection] + ) + + response2 = client.responses.create( + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] + input=[ + { + "role": "developer", + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] + }, + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": query # $ Alert[py/prompt-injection] + } + ] + } + ] + ) + + response3 = await async_client.responses.create( + instructions="Talks like a " + persona, # $ Alert[py/prompt-injection] + input=query, # $ Alert[py/prompt-injection] + ) + + async with client.realtime.connect(model="gpt-realtime") as connection: + await connection.conversation.item.create( + item={ + "type": "message", + "role": role, + "content": [ + { + "type": "input_text", + "text": query # $ Alert[py/prompt-injection] + } + ], + } + ) + + completion1 = client.chat.completions.create( + messages=[ + { + "role": "developer", + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] + }, + { + "role": "user", + "content": query, # $ Alert[py/prompt-injection] + }, + { + "role": role, + "content": query, # $ Alert[py/prompt-injection] + } + ] + ) + + completion2 = azure_client.chat.completions.create( + messages=[ + { + "role": "developer", + "content": "Talk like a " + persona # $ Alert[py/prompt-injection] + }, + { + "role": "user", + "content": query, # $ Alert[py/prompt-injection] + } + ] + ) + + assistant = client.beta.assistants.create( + name="Test Agent", + model="gpt-4.1", + instructions="Talks like a " + persona # $ Alert[py/prompt-injection] + )