diff --git a/.tekton/.currency/docs/report.md b/.tekton/.currency/docs/report.md index 421700e95..ddadb38b7 100644 --- a/.tekton/.currency/docs/report.md +++ b/.tekton/.currency/docs/report.md @@ -1,30 +1,30 @@ ##### This page is auto-generated. Any change will be overwritten after the next sync. Please apply changes directly to the files in the [python tracer](https://github.com/instana/python-sensor) repo. ## Python supported packages and versions -| Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Cloud Native | -|:---------------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------| -| ASGI | 45-days | No | 3.0 | 3.0 | Yes | No | -| Celery | 45-days | No | 5.4.0 | 5.4.0 | Yes | No | -| Django | 45-days | No | 5.1.1 | 5.1.1 | Yes | No | -| FastAPI | 45-days | No | 0.115.0 | 0.115.0 | Yes | No | -| Flask | 45-days | No | 3.0.3 | 3.0.3 | Yes | No | -| Pyramid | 45-days | No | 2.0.2 | 2.0.2 | Yes | No | -| Sanic | On demand | No | 24.6.0 | 24.6.0 | Yes | No | -| Starlette | 45-days | No | 0.38.6 | 0.39.2 | No | No | -| Tornado | 45-days | No | 6.4.1 | 6.4.1 | Yes | No | -| Webapp2 | On demand | No | 2.5.2 | 2.5.2 | Yes | No | -| WSGI | 0-day | Yes | 1.0.1 | 1.0.1 | Yes | No | -| Aiohttp | 45-days | No | 3.10.8 | 3.10.8 | Yes | No | -| Asynqp | Deprecated | No | 0.6 | 0.6 | Yes | No | -| Boto3 | 45-days | No | 1.35.33 | 1.35.33 | Yes | Yes | -| Google-cloud-pubsub | 45-days | No | 2.25.2 | 2.25.2 | Yes | Yes | -| Google-cloud-storage | 45-days | No | 2.18.2 | 2.18.2 | Yes | Yes | -| Grpcio | 45-days | No | 1.66.2 | 1.66.2 | Yes | Yes | -| Mysqlclient | 45-days | No | 2.2.4 | 2.2.4 | Yes | Yes | -| Pika | 45-days | No | 1.3.2 | 1.3.2 | Yes | No | -| PyMySQL | 45-days | No | 1.1.1 | 1.1.1 | Yes | Yes | -| Pymongo | 45-days | No | 4.10.1 | 4.10.1 | Yes | Yes | -| Psycopg2 | 45-days | No | 2.9.9 | 2.9.9 | Yes | No | -| Redis | 45-days | No | 5.1.1 | 5.1.1 | Yes | Yes | -| Requests | 45-days | No | 2.32.3 | 2.32.3 | Yes | Yes | -| SQLAlchemy | 45-days | No | 2.0.35 | 2.0.35 | Yes | Yes | -| Urllib3 | 45-days | No | 2.2.3 | 2.2.3 | Yes | No | \ No newline at end of file +| Package name | Support Policy | Beta version | Last Supported Version | Latest version | Up-to-date | Release date | Latest Version Published At | Days behind | Cloud Native | +|:---------------------|:-----------------|:---------------|:-------------------------|:-----------------|:-------------|:---------------|:------------------------------|:--------------|:---------------| +| ASGI | 45-days | No | 3.0 | 3.0 | Yes | 2019-03-04 | 2019-03-04 | 0 day/s | No | +| Celery | 45-days | No | 5.4.0 | 5.4.0 | Yes | 2024-04-17 | 2024-04-17 | 0 day/s | No | +| Django | 45-days | No | 5.1.6 | 5.1.6 | Yes | 2025-02-05 | 2025-02-05 | 0 day/s | No | +| FastAPI | 45-days | No | 0.115.11 | 0.115.11 | Yes | 2025-03-01 | 2025-03-01 | 0 day/s | No | +| Flask | 45-days | No | 3.1.0 | 3.1.0 | Yes | 2024-11-13 | 2024-11-13 | 0 day/s | No | +| Pyramid | 45-days | No | 2.0.2 | 2.0.2 | Yes | 2023-08-25 | 2023-08-25 | 0 day/s | No | +| Sanic | On demand | No | 24.12.0 | 24.12.0 | Yes | 2024-12-31 | 2024-12-31 | 0 day/s | No | +| Starlette | 45-days | No | 0.46.0 | 0.46.0 | Yes | 2025-02-22 | 2025-02-22 | 0 day/s | No | +| Tornado | 45-days | No | 6.4.2 | 6.4.2 | Yes | 2024-11-22 | 2024-11-22 | 0 day/s | No | +| Webapp2 | On demand | No | 2.5.2 | 2.5.2 | Yes | 2012-09-28 | 2012-09-28 | 0 day/s | No | +| WSGI | 0-day | Yes | 1.0.1 | 1.0.1 | Yes | 2010-09-26 | 2010-09-26 | 0 day/s | No | +| Aiohttp | 45-days | No | 3.11.13 | 3.11.13 | Yes | 2025-02-24 | 2025-02-24 | 0 day/s | No | +| Asynqp | Deprecated | No | 0.6 | 0.6 | Yes | 2019-01-20 | 2019-01-20 | 0 day/s | No | +| Boto3 | 45-days | No | 1.37.5 | 1.37.5 | Yes | 2025-03-03 | 2025-03-03 | 0 day/s | Yes | +| Google-cloud-pubsub | 45-days | No | 2.28.0 | 2.28.0 | Yes | 2025-01-30 | 2025-01-30 | 0 day/s | Yes | +| Google-cloud-storage | 45-days | No | 3.1.0 | 3.1.0 | Yes | 2025-02-28 | 2025-02-28 | 0 day/s | Yes | +| Grpcio | 45-days | No | 1.71.0rc2 | 1.70.0 | Yes | 2025-01-23 | 2025-02-24 | 0 day/s | Yes | +| Mysqlclient | 45-days | No | 2.2.7 | 2.2.7 | Yes | 2025-01-10 | 2025-01-10 | 0 day/s | Yes | +| Pika | 45-days | No | 1.3.2 | 1.3.2 | Yes | 2023-05-05 | 2023-05-05 | 0 day/s | No | +| PyMySQL | 45-days | No | 1.1.1 | 1.1.1 | Yes | 2024-05-21 | 2024-05-21 | 0 day/s | Yes | +| Pymongo | 45-days | No | 4.11.2 | 4.11.2 | Yes | 2025-03-03 | 2025-03-03 | 0 day/s | Yes | +| Psycopg2 | 45-days | No | 2.9.10 | 2.9.10 | Yes | 2024-10-16 | 2024-10-16 | 0 day/s | No | +| Redis | 45-days | No | 5.2.1 | 5.2.1 | Yes | 2024-12-06 | 2024-12-06 | 0 day/s | Yes | +| Requests | 45-days | No | 2.32.3 | 2.32.3 | Yes | 2024-05-29 | 2024-05-29 | 0 day/s | Yes | +| SQLAlchemy | 45-days | No | 2.0.38 | 2.0.38 | Yes | 2025-02-06 | 2025-02-06 | 0 day/s | Yes | +| Urllib3 | 45-days | No | 2.3.0 | 2.3.0 | Yes | 2024-12-22 | 2024-12-22 | 0 day/s | No | \ No newline at end of file diff --git a/.tekton/.currency/scripts/generate_report.py b/.tekton/.currency/scripts/generate_report.py index ad9b3869b..20dc68899 100644 --- a/.tekton/.currency/scripts/generate_report.py +++ b/.tekton/.currency/scripts/generate_report.py @@ -1,6 +1,7 @@ # Standard Libraries import json import re +from datetime import datetime import pandas as pd @@ -13,6 +14,7 @@ JSON_FILE = "resources/table.json" REPORT_FILE = "docs/report.md" PIP_INDEX_URL = "https://pypi.org/pypi" +PEP_BASE_URL = "https://peps.python.org/" SPEC_MAP = { "ASGI": "https://asgi.readthedocs.io/en/latest/specs/main.html", @@ -20,37 +22,96 @@ } -def get_upstream_version(dependency): +def estimate_days_behind(release_date): + return datetime.today() - datetime.strptime(release_date, "%Y-%m-%d") + + +def get_upstream_version(dependency, last_supported_version): """Get the latest version available upstream""" + last_supported_version_release_date = "Not found" if dependency in SPEC_MAP: # webscrape info from official website - pattern = "(\d+\.\d+\.?\d*)" + version_pattern = "(\d+\.\d+\.?\d*)" + latest_version_release_date = "" url = SPEC_MAP[dependency] page = requests.get(url) soup = BeautifulSoup(page.text, "html.parser") # ASGI if "asgi" in url: - text = ( - soup.find(id="version-history") - .findChild("li", string=re.compile(pattern)) - .text - ) + all_versions = soup.find(id="version-history").find_all("li") + pattern = re.compile(r"([\d.]+) \((\d{4}-\d{2}-\d{2})\)") + latest_version, latest_version_release_date = pattern.search( + all_versions[0].text + ).groups() + for li in all_versions: + match = pattern.search(li.text) + if match: + version, date = match.groups() + if version == last_supported_version: + last_supported_version_release_date = date + break # WSGI else: - tag = soup.find(id="numerical-index").find_all( + all_versions = soup.find(id="numerical-index").find_all( "a", string=re.compile("Web Server Gateway Interface") - )[-1] - text = tag.text - res = re.search(pattern, text) - return res[1] + ) + latest_version = re.search(version_pattern, all_versions[-1].text).group() + + for a in all_versions: + pep_link = PEP_BASE_URL + a.get("href").split("..")[1] + response = requests.get(pep_link) + soup = BeautifulSoup(response.text, "html.parser") + version = re.search(version_pattern, a.text).group() + pep_page_metadata = soup.find("dl") + + if pep_page_metadata and version in [ + latest_version, + last_supported_version, + ]: + metadata_fields = pep_page_metadata.find_all("dt") + metadata_values = pep_page_metadata.find_all("dd") + + for dt, dd in zip(metadata_fields, metadata_values): + if "Created" in dt.text: + release_date = dd.text.strip() + release_date_as_datetime = datetime.strptime( + release_date, "%d-%b-%Y" + ) + if version == latest_version: + latest_version_release_date = ( + release_date_as_datetime.strftime("%Y-%m-%d") + ) + if version == last_supported_version: + last_supported_version_release_date = ( + release_date_as_datetime.strftime("%Y-%m-%d") + ) + return ( + latest_version, + latest_version_release_date, + last_supported_version_release_date, + ) else: # get info using PYPI API response = requests.get(f"{PIP_INDEX_URL}/{dependency}/json") response_json = response.json() latest_version = response_json["info"]["version"] - return latest_version + release_time = response_json["releases"][latest_version][-1][ + "upload_time_iso_8601" + ] + latest_version_release_date = datetime.fromisoformat(release_time) + formatted_release_date = latest_version_release_date.strftime("%Y-%m-%d") + for version, release_info in response_json["releases"].items(): + if version == last_supported_version: + release_time = release_info[-1]["upload_time_iso_8601"] + release_date = datetime.fromisoformat(release_time) + last_supported_version_release_date = release_date.strftime("%Y-%m-%d") + return ( + latest_version, + formatted_release_date, + last_supported_version_release_date, + ) def get_last_supported_version(tekton_ci_output, dependency): @@ -67,14 +128,18 @@ def get_last_supported_version(tekton_ci_output, dependency): return last_supported_version[1] -def isUptodate(last_supported_version, latest_version): +def is_up_to_date( + last_supported_version, latest_version, last_supported_version_release_date +): """Check if the supported package is up-to-date""" if Version(last_supported_version) >= Version(latest_version): up_to_date = "Yes" + days_behind = 0 else: up_to_date = "No" + days_behind = estimate_days_behind(last_supported_version_release_date) - return up_to_date + return up_to_date, days_behind def get_taskruns(namespace, task_name, taskrun_filter): @@ -144,7 +209,7 @@ def get_tekton_ci_output(): core_v1_client = client.CoreV1Api() task_name = "python-tracer-unittest-gevent-starlette-task" - taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" + taskrun_filter = lambda tr: tr["status"]["conditions"][0]["type"] == "Succeeded" # noqa: E731 starlette_taskruns = get_taskruns(namespace, task_name, taskrun_filter) tekton_ci_output = process_taskrun_logs( @@ -152,7 +217,7 @@ def get_tekton_ci_output(): ) task_name = "python-tracer-unittest-googlecloud-task" - taskrun_filter = ( + taskrun_filter = ( # noqa: E731 lambda tr: tr["metadata"]["name"].endswith("unittest-googlecloud-0") and tr["status"]["conditions"][0]["type"] == "Succeeded" ) @@ -163,7 +228,7 @@ def get_tekton_ci_output(): ) task_name = "python-tracer-unittest-default-task" - taskrun_filter = ( + taskrun_filter = ( # noqa: E731 lambda tr: tr["metadata"]["name"].endswith("unittest-default-3") and tr["status"]["conditions"][0]["type"] == "Succeeded" ) @@ -195,11 +260,23 @@ def main(): else: last_supported_version = item["Last Supported Version"] - latest_version = get_upstream_version(package) + latest_version, release_date, last_supported_version_release_date = ( + get_upstream_version(package, last_supported_version) + ) - up_to_date = isUptodate(last_supported_version, latest_version) + up_to_date, days_behind = is_up_to_date( + last_supported_version, latest_version, last_supported_version_release_date + ) - item.update({"Latest version": latest_version, "Up-to-date": up_to_date}) + item.update( + { + "Latest version": latest_version, + "Up-to-date": up_to_date, + "Release date": release_date, + "Latest Version Published At": last_supported_version_release_date, + "Days behind": f"{days_behind} day/s", + } + ) # Create a DataFrame from the list of dictionaries df = pd.DataFrame(items)