Sphinx doc + GitHub pages deployment

ShellCode33 · ShellCode33 · commit cf55b390efff · 2020-07-09T00:17:20.000+02:00
diff --git a/.travis.yml b/.travis.yml
@@ -13,7 +13,9 @@ python:
 - '3.8'
 - 'pypy3'
 
-script: python -m unittest tests/tests.py
+script:
+  - python -m unittest tests/tests.py
+  - make -C docs html
 
 deploy:
   - provider: releases
@@ -27,6 +29,14 @@ deploy:
     skip_cleanup: true
     api_key:
       secure: IcX91IbJikIWp5p9AgJWKVpVIhpBZdpstQmz7hES9BnGdh1o/cLcIHcJGhpJXqSQ+kEspxw5DHMhKYFcKShWmj1TyG4TI1eCNycJ/F8f2d4TRBvucZxNfiPZF7GmxZrWEze6iFJ4p3KPBzwQ2zfggxYwbyEL5d73vfLcVc3nQDpoanRvF94EDobMuAeKk5P01OGRU2IK4pMi1ESOolUZr2XKiNaTgNqol5np/XI6Gte+Mlx8bo+AcVKCC+mSSVotrTsPsRErma9sVF9OYf8OH0jySwAt2tqvWRAC66R+inJMbH1qJdJ1EsO2hyF3N7pe9oIGJZkRqxd7Z20MTDg8zYCydbNQB/bsLZeA1fLmt5uQ3+AmIGnia27v7h5tGd5oeceMUbmXtYbX5midfvDXnpZcpqUE/8aUiN/Lq0Lr6qGGiSjvlynEYBWfMY5W4yLhpsKryTn/oaeBMAoMg2IY4BFkSZ+Evtu1RQe5EloYzwT/QjJ2LVO6Rb0ORZpmXUZcuFJOLoISFDqjaI5OpamDuxwKzvR6OGwq+sFYTR42rLN4YZ9NVbUb5AqkKdwqvo8qRiEtivkokbcNwRtMSgEnvD1TSLArYHQdY9gvLNP8CTyW6qO+0Wt1/DianqjYhsk3fBBHijfRbBBRqfyYNmipLm0cPZ1VW4OeBRF+A1OumRc=
+  - provider: pages
+    skip_cleanup: true
+    local_dir: docs/_build/html/
+    on:
+      repo: ShellCode33/CredSLayer
+      tags: true
+    github_token:
+      secure: IcX91IbJikIWp5p9AgJWKVpVIhpBZdpstQmz7hES9BnGdh1o/cLcIHcJGhpJXqSQ+kEspxw5DHMhKYFcKShWmj1TyG4TI1eCNycJ/F8f2d4TRBvucZxNfiPZF7GmxZrWEze6iFJ4p3KPBzwQ2zfggxYwbyEL5d73vfLcVc3nQDpoanRvF94EDobMuAeKk5P01OGRU2IK4pMi1ESOolUZr2XKiNaTgNqol5np/XI6Gte+Mlx8bo+AcVKCC+mSSVotrTsPsRErma9sVF9OYf8OH0jySwAt2tqvWRAC66R+inJMbH1qJdJ1EsO2hyF3N7pe9oIGJZkRqxd7Z20MTDg8zYCydbNQB/bsLZeA1fLmt5uQ3+AmIGnia27v7h5tGd5oeceMUbmXtYbX5midfvDXnpZcpqUE/8aUiN/Lq0Lr6qGGiSjvlynEYBWfMY5W4yLhpsKryTn/oaeBMAoMg2IY4BFkSZ+Evtu1RQe5EloYzwT/QjJ2LVO6Rb0ORZpmXUZcuFJOLoISFDqjaI5OpamDuxwKzvR6OGwq+sFYTR42rLN4YZ9NVbUb5AqkKdwqvo8qRiEtivkokbcNwRtMSgEnvD1TSLArYHQdY9gvLNP8CTyW6qO+0Wt1/DianqjYhsk3fBBHijfRbBBRqfyYNmipLm0cPZ1VW4OeBRF+A1OumRc=
   - provider: pypi
     distributions: sdist bdist_wheel
     skip_existing: true
@@ -35,4 +45,4 @@ deploy:
     skip_cleanup: true
     user: __token__
     password:
-      secure: nbDYdgllAoXpNV0tXLYAp8g1gGEboJ6fhvbC+H2XW66CciMfX5NfhM2mM3RUHIjJOLVsffuj7bg92QhwC72zK6dB/tNXOxw2U4Cs4/+h0hITzKfD2VkCgfVGF3XGR5xlB1hV1ZJAo+74qcNRd3KoaVhLRVpt174iGlXcQaHR0TdKGCExPedbviu5nNZD7Gde1cSvKnXfPk8ZVHzYU0O5X6HVJdOutFV7oAW1uqmEL3/AGc5Gomgfdl0O8AL8Al7SlFGX8Kts5zPE2h11SmGfIJuvkLbJRZwQ8UlHreT3rKpQuGCukXXFQbx3tRmWIb9zM2Np93QZsQ9qFQbyIekJvY9kqp9Lx8sq6sYQcbvXb9fp1+18+m4J5EfJ8D1Z1AgrjOaD2hlqxFwD91yZnWA0AY4yaKGU/GU18t1HTgT3I27AHMCj7OqR/CcRt9h/AYdrPiCzLGWqzvjOX2ayVsquVrkP39z5HFi2pJOp0fuuwMGYrVQ0GRYqMmZbiGiRayJFQXoZ9jY67RaffdDIzapw6pAuKd5bjNe0P6MpAyE7qk5Kz1XnCwNjhLzAFfglI3J5hVkoDzf8S6FFWyhYvWDWtxQ6wJqAzNIAiXJ2hI/5nxEkmKyLMk6dvcJdC9tjMFyqsLrQEbTAu04yJlYHWaAJ0F/HTEVo8Aw4cQdDpNdarfc=
+      secure: nbDYdgllAoXpNV0tXLYAp8g1gGEboJ6fhvbC+H2XW66CciMfX5NfhM2mM3RUHIjJOLVsffuj7bg92QhwC72zK6dB/tNXOxw2U4Cs4/+h0hITzKfD2VkCgfVGF3XGR5xlB1hV1ZJAo+74qcNRd3KoaVhLRVpt174iGlXcQaHR0TdKGCExPedbviu5nNZD7Gde1cSvKnXfPk8ZVHzYU0O5X6HVJdOutFV7oAW1uqmEL3/AGc5Gomgfdl0O8AL8Al7SlFGX8Kts5zPE2h11SmGfIJuvkLbJRZwQ8UlHreT3rKpQuGCukXXFQbx3tRmWIb9zM2Np93QZsQ9qFQbyIekJvY9kqp9Lx8sq6sYQcbvXb9fp1+18+m4J5EfJ8D1Z1AgrjOaD2hlqxFwD91yZnWA0AY4yaKGU/GU18t1HTgT3I27AHMCj7OqR/CcRt9h/AYdrPiCzLGWqzvjOX2ayVsquVrkP39z5HFi2pJOp0fuuwMGYrVQ0GRYqMmZbiGiRayJFQXoZ9jY67RaffdDIzapw6pAuKd5bjNe0P6MpAyE7qk5Kz1XnCwNjhLzAFfglI3J5hVkoDzf8S6FFWyhYvWDWtxQ6wJqAzNIAiXJ2hI/5nxEkmKyLMk6dvcJdC9tjMFyqsLrQEbTAu04yJlYHWaAJ0F/HTEVo8Aw4cQdDpNdarfc=
diff --git a/docs/Makefile b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css
@@ -0,0 +1,5 @@
+@import 'theme.css';
+
+.wy-side-nav-search {
+    background-color: darkred;
+}
diff --git a/docs/conf.py b/docs/conf.py
@@ -0,0 +1,70 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+
+import sphinx_rtd_theme
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'CredSLayer'
+copyright = '2020, ShellCode'
+author = 'ShellCode'
+
+# The full version, including alpha/beta/rc tags
+release = '1.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
+    'sphinx_rtd_theme',
+    'sphinxarg.ext',
+    'sphinx.ext.autosectionlabel'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+html_theme_options = {
+    'style_nav_header_background': 'darkred',
+    'collapse_navigation': True
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+add_module_names = False
diff --git a/docs/contribute/code-explanations.rst b/docs/contribute/code-explanations.rst
@@ -0,0 +1,12 @@
+Code explanations
+=================
+
+You will find here concepts you must be aware of to get involved.
+You do not need to be a Python expert to contribute,
+CredSLayer has been made in a way that is fairly easy to extend.
+
+.. toctree::
+    :maxdepth: 4
+
+    code/sessions
+    code/parsers
diff --git a/docs/contribute/code/parsers.rst b/docs/contribute/code/parsers.rst
@@ -0,0 +1,4 @@
+Parsers
+========
+
+.. automodule:: credslayer.parsers
diff --git a/docs/contribute/code/sessions.rst b/docs/contribute/code/sessions.rst
@@ -0,0 +1,17 @@
+Sessions
+========
+
+CredSLayer's sessions is probably the most important concept, it helps you create a context between loads of packets you "receive".
+It will hold credentials you find and enables parsers to keep variables across per-packet analysis.
+
+For example in telnet, data are transmitted line by line, and sometimes character by character.
+So the parser gathers those pieces of information and store them in a session variable that will be available to further packets belonging to the same session :
+
+.. code-block:: python
+
+    session["data_being_built"] += data
+
+The lines that follow are directly taken from the code's documentation and explain more in depth how sessions work.
+
+.. automodule:: credslayer.core.session
+    :members:
diff --git a/docs/contribute/create-parser.rst b/docs/contribute/create-parser.rst
@@ -0,0 +1,79 @@
+Tutorial : new parser
+=====================
+
+Create a new file in ``credslayer/parsers/`` named after the protocol you want to analyse. Usually you want to name your file the same as the `Protocol` column in wireshark.
+
+.. image:: wireshark_protocol_column.png
+    :width: 800
+    :alt: Wireshark protocol column
+
+But sometimes it won't work, so you might wanna check the name of the protocol by searching it using the following command line :
+
+.. code-block:: none
+
+    $ tshark -G protocols | grep -i SEARCH_HERE | awk '{print $NF}'
+
+
+You now have to implement the following function that CredSLayer will automatically call for each incoming packet of your choosen protocol :
+
+.. autofunction:: credslayer.parsers.ftp.analyse
+
+Let's just take the ftp parser as an example, just read the comments :)
+
+.. code-block:: python
+
+    # coding: utf-8
+
+    from pyshark.packet.layer import Layer
+
+    from credslayer.core import logger
+    from credslayer.core.session import Session
+
+
+    # Remember this function is called for every single packet that belong to the protocol
+    def analyse(session: Session, layer: Layer):
+
+        # You can use the pretty_print method to show what's in the packet
+        # layer.pretty_print()
+
+        # Just an alias of credentials_being_built
+        current_creds = session.credentials_being_built
+
+        # We check (using pyshark's layer object) if the packet contains a "reponse_code" field.
+        # This is the name tshark gave to the portion of the packet holding the response code.
+        if hasattr(layer, "response_code"):
+            code = int(layer.response_code)
+
+            # Code 230 means authentication has been successful
+            if code == 230 and current_creds.username:
+                # That's how you log credentials you've just found
+                logger.found(session, "credentials found: {} -- {}".format(current_creds.username, current_creds.password))
+
+                # That's how you tell CredsLayer to remove the context and credentials currently being built.
+                # You found everything you needed, you're done with everything you saved in the session.
+                session.validate_credentials()
+
+            # Code 430 means credentials were wrong
+            elif code == 430:
+                # That's also how you tell CredsLayer to remove the context and credentials currently being built.
+                # You just found out credentials were wrong, you're not interested in what you saved in the session.
+                session.invalidate_credentials_and_clear_session()
+
+        # We check (using pyshark's layer object) if the packet contains a "request_command" field,
+        # This is the name tshark gave to the portion of the packet holding the FTP command.
+        elif hasattr(layer, "request_command"):
+            command = layer.request_command
+
+            # The username is sent next to the "USER" command.
+            # We store the username in `current_creds`, which is the alias for the credentials being built in the session.
+            if command == "USER":
+                current_creds.username = layer.request_arg
+
+            # Idem with the password
+            elif command == "PASS":
+                current_creds.password = layer.request_arg
+
+I really encourage you to read about :ref:`Sessions` as this is what enables you to build a persistent context across each call to your *analyse* function.
+For example in FTP, the username is first sent in a packet, then the password in another one. Thanks to the :ref:`Sessions` object, you will be able to keep track
+of some valuable data. Then somehow (depending on the protocol you are analysing) you might receive an indicator of success or failure, in FTP codes 230 and 430 fulfill that purpose.
+When you find such an indicator, you must call ``session.invalidate_credentials_and_clear_session`` or ``session.validate_credentials`` to tell CredSLayer you're done with those credentials.
diff --git a/docs/contribute/setup.rst b/docs/contribute/setup.rst
@@ -0,0 +1,43 @@
+Environment setup
+=================
+
+First you must `fork the repository <https://github.com/ShellCode33/CredSLayer/fork>`_.
+Then clone your personal repo and create a new branch based on develop :
+
+.. code-block:: none
+
+    $ git clone https://github.com/[YourUsername]/CredSLayer
+    $ cd CredSLayer/
+    $ git checkout -b [Choose a name] develop
+
+To prevent any dependencies conflicts, it's recommended to create a `virtualenv <https://docs.python.org/fr/3/library/venv.html>`_.
+CredSLayer supports Python 3.5 and above, so in order to make sure your code is compatible, I suggest you install this version of Python and create a virtualenv with it :
+
+.. code-block:: none
+
+    $ python3.5 -m venv venv
+    $ source venv/bin/activate
+
+Then install the project in "dev mode" :
+
+.. code-block:: none
+
+    $ pip install -e .
+
+At this point, ``credslayer`` should be available from the command line (only when you're inside the virtualenv). The command will automatically use the latest changes you make to the code.
+
+Once you're done with your modifications, make sure your code didn't break anything by running the unit tests:
+
+.. code-block:: none
+
+    $ python -m unittest tests/tests.py
+
+If there's no failure, add, commit and push your code :
+
+.. code-block:: none
+
+    $ git add path/to/updated/file
+    $ git commit -m "short description of your changes"
+    $ git push
+
+Then go to GitHub and create a pull request. I will review it and decide whether or not it's worth integrating it.
diff --git a/docs/contribute/wireshark_protocol_column.png b/docs/contribute/wireshark_protocol_column.png
diff --git a/docs/example.png b/docs/example.png
diff --git a/docs/getting-started/installation.rst b/docs/getting-started/installation.rst
@@ -0,0 +1,17 @@
+Installation
+============
+
+The recommended way is to pull it from PyPI repository :
+
+.. parsed-literal::
+    $ [sudo] pip install credslayer
+
+You can also install it from source :
+
+.. parsed-literal::
+    $ git clone https://github.com/ShellCode33/CredSLayer
+    $ cd CredSLayer/
+    $ [sudo] python setup.py install
+
+.. note::
+    If you want to try brand new features not released yet, from source you can switch to the develop branch using ``git checkout develop`` before installing.
diff --git a/docs/getting-started/limitations.rst b/docs/getting-started/limitations.rst
@@ -0,0 +1,31 @@
+Limitations
+===========
+
+One of the biggest issue is probably performance. CredSLayer relies on pyshark (a wrapper arround tshark) and both are written in Python. I personally find it to be a great choice for a community project because it's easy to read and many people know how to write Python code.
+But unfortunately Python is pretty slow when it comes to processing a lot of data. Therefore CredSLayer will obviously not work very well if you run it on a backbone, but hopefully people will find it to be enough for most of their needs.
+
+To prevent false positives, false negatives and performance issues, **Wireshark identifies most protocols based on their default port number**. It means that if, for example, a telnet server is running on the port 1337 (which is not default) Wireshark will not be able to identify it and will therefore not process it.
+
+However, Wireshark implements mechanisms called **heuristic dissectors**, it enables Wireshark (and by extension CredSLayer) to identify **some** protocols on other ports than the default one. Here's what Wireshark's documentation says about heuristic dissectors :
+
+.. code-block:: text
+
+    When Wireshark "receives" a packet, it has to find the right dissector to
+    start decoding the packet data. Often this can be done by known conventions,
+    e.g. the Ethernet type 0x0800 means "IP on top of Ethernet" - an easy and
+    reliable match for Wireshark.
+
+    Unfortunately, these conventions are not always available, or (accidentally
+    or knowingly) some protocols don't care about those conventions and "reuse"
+    existing "magic numbers / tokens".
+
+    For example TCP defines port 80 only for the use of HTTP traffic. But, this
+    convention doesn't prevent anyone from using TCP port 80 for some different
+    protocol, or on the other hand using HTTP on a port number different than 80.
+
+    To solve this problem, Wireshark introduced the so called heuristic dissector
+    mechanism to try to deal with these problems.
+
+Thanks to this, some easy-to-identify-protocols can be detected on any port. This is the case of HTTP which is pretty easy to spot because its payloads always start with a method (GET, POST, OPTIONS, etc.).
+
+To enable CredSLayer to still be able to analyse "hidden" protocols, the CLI has a ``--map`` parameter which enables you to map a (range of) port(s) to a specific protocol. This way you will be able to sniff credentials going to a specific service on a specific port you are aware of.
diff --git a/docs/getting-started/usage-cli.rst b/docs/getting-started/usage-cli.rst
@@ -0,0 +1,6 @@
+The command line
+================
+
+.. argparse::
+    :ref: credslayer.credslayer.build_argument_parser
+    :prog: credslayer
diff --git a/docs/getting-started/usage-own-project.rst b/docs/getting-started/usage-own-project.rst
@@ -0,0 +1,28 @@
+Use in your own project
+=======================
+
+CredSLayer is available in PyPI repository, you can simply add ``credslayer`` to your ``requirements.txt`` file.
+
+The easiest way to retrieve credentials from a pcap is as follow :
+
+.. code-block:: python
+
+    from credslayer.manager import process_pcap
+
+    if __name__ == "__main__":
+        credentials = process_pcap("my_capture.pcap").get_list_of_all_credentials()
+
+        for creds in credentials:
+            print(creds)
+
+I you want to perform a live capture and process credentials as they come, you can use a callback as follow :
+
+.. code-block:: python
+
+    from credslayer.manager import active_processing
+
+    def process_found_credentials(credentials: Credentials):
+        print("Found:", credentials)
+
+    if __name__ == "__main__":
+        active_processing("wlp2s0", callback=process_found_credentials)
diff --git a/docs/index.rst b/docs/index.rst
diff --git a/docs/make.bat b/docs/make.bat
diff --git a/requirements.txt b/requirements.txt

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +Parsers
 +========
++
 +.. automodule:: credslayer.parsers