POC for Agent Lite: make Python and integrations optional

omnibus/config/projects/agent.rb

@@ -267,6 +267,22 @@
 # will package is the correct one
 disable_version_manifest do_package
 
+# For lite flavor, create separate Python artifacts
+if flavor == 'lite' && do_build
+  # Create Python runtime artifact
+  if Dir.exist?("#{install_dir}/python-runtime")
+    extra_package_file "#{install_dir}/python-runtime" => "datadog-lite-agent-python-runtime"
+  end
+
+  # Create Python integrations artifact  
+  if Dir.exist?("#{install_dir}/python-integrations")
+    extra_package_file "#{install_dir}/python-integrations" => "datadog-lite-agent-python-integrations"
+  end
+
+  # Exclude Python artifacts from main package
+  exclude 'python-runtime'
+  exclude 'python-integrations'
+end
 
 if linux_target?
   extra_package_file "#{output_config_dir}/etc/datadog-agent/"

omnibus/config/software/datadog-agent-dependencies.rb

@@ -22,7 +22,10 @@
 dependency 'jmxfetch'
 
 # Used for memory profiling with the `status py` agent subcommand
-dependency 'pympler'
+# For lite flavor, pympler goes with Python runtime artifacts
+unless ENV['AGENT_FLAVOR'] == 'lite'
+  dependency 'pympler'
+end
 
 dependency "systemd" if linux_target?
 
@@ -31,7 +34,13 @@
 # Include traps db file in snmp.d/traps_db/
 dependency 'snmp-traps'
 
-dependency 'datadog-agent-integrations-py3'
+# Build Python integrations but package separately for lite flavor
+if ENV['AGENT_FLAVOR'] == 'lite'
+  dependency 'datadog-agent-integrations-py3-lite'
+  dependency 'datadog-agent-python-runtime-lite'
+else
+  dependency 'datadog-agent-integrations-py3'
+end
 
 
 # Additional software

omnibus/config/software/datadog-agent-integrations-py3-lite.rb

@@ -0,0 +1,95 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https:#www.datadoghq.com/).
+# Copyright 2016-present Datadog, Inc.
+
+require './lib/ostools.rb'
+require 'json'
+
+name 'datadog-agent-integrations-py3-lite'
+
+license "BSD-3-Clause"
+license_file "./LICENSE"
+
+dependency 'datadog-agent-integrations-py3-dependencies'
+
+python_version = "3.13"
+
+relative_path 'integrations-core'
+whitelist_file "embedded/lib/python#{python_version}/site-packages/.libsaerospike"
+whitelist_file "embedded/lib/python#{python_version}/site-packages/aerospike.libs"
+whitelist_file "embedded/lib/python#{python_version}/site-packages/psycopg_binary.libs"
+whitelist_file "embedded/lib/python#{python_version}/site-packages/pymqi"
+
+source git: 'https://github.com/DataDog/integrations-core.git'
+
+integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION']
+if integrations_core_version.nil? || integrations_core_version.empty?
+  integrations_core_version = 'master'
+end
+default_version integrations_core_version
+
+build do
+  # Set the PYTHONPATH and PYTHONHOME
+  env = {
+    "PYTHONPATH" => "#{install_dir}/embedded/lib/python#{python_version}/site-packages",
+    "PYTHONHOME" => "#{install_dir}/embedded",
+  }
+
+  # For lite flavor, install integrations to separate directory
+  integrations_dir = "#{install_dir}/python-integrations"
+  mkdir integrations_dir
+  mkdir "#{integrations_dir}/lib/python#{python_version}/site-packages"
+
+  env["PYTHONPATH"] = "#{integrations_dir}/lib/python#{python_version}/site-packages"
+
+  pip_install_args = "#{windows_safe_path(install_dir)}\\embedded3\\python.exe -m pip install --no-deps --no-binary=:all:" if windows_target?
+  pip_install_args = "#{install_dir}/embedded/bin/pip3 install --no-deps --no-binary=:all:" unless windows_target?
+
+  patch :source => 'create-regex-at-runtime.patch', :env => env if ohai['platform'] == "windows" && !arm64_target?
+
+  # Get static requirements
+  env_script_path = "#{windows_safe_path(install_dir)}/embedded3/Scripts/Activate.ps1"
+  env_script_path = ". #{install_dir}/embedded/bin/activate" unless windows_target?
+
+  requirements_file = 'omnibus/config/templates/datadog-agent-integrations-py3/static_requirements.txt'
+
+  if windows_target?
+    requirements_file = 'omnibus\\config\\templates\\datadog-agent-integrations-py3\\static_requirements.txt'
+  end
+
+  erb :dest => "#{Dir.pwd}/static_requirements.txt",
+      :source => "#{windows_safe_path(Dir.pwd)}/#{requirements_file}.erb",
+      :mode => 0755,
+      :vars => { :python_version => python_version, :requirements => {}, :static_reqs_in_requirements => [] }
+
+  # Use the static requirements file to install base requirements to python-integrations dir
+  if windows_target?
+    install_cmd = "#{pip_install_args} --target #{integrations_dir}\\lib\\python#{python_version}\\site-packages"
+  else
+    install_cmd = "#{pip_install_args} --target #{integrations_dir}/lib/python#{python_version}/site-packages"
+  end
+
+  command "#{install_cmd} -r static_requirements.txt", :env => env
+
+  # Install integrations to separate directory
+  # Use smaller core set for lite flavor
+  core_integrations = %w[cpu disk io memory network uptime]
+
+  core_integrations.each do |integration|
+    if File.exist?("#{integration}/setup.py")
+      command "#{install_cmd} ./#{integration}", :env => env, :cwd => "#{project_dir}"
+    end
+  end
+
+  # Create marker file to indicate this is a lite integrations package
+  command "echo 'lite' > #{integrations_dir}/AGENT_FLAVOR"
+
+  # Set proper permissions
+  if ohai['platform'] == "windows"
+    command "icacls #{windows_safe_path(integrations_dir)} /T /Q /C /RESET"
+  else
+    command "find #{integrations_dir} -type f -exec chmod 644 {} +"
+    command "find #{integrations_dir} -type d -exec chmod 755 {} +"
+  end
+end

omnibus/config/software/datadog-agent-python-runtime-lite.rb

@@ -0,0 +1,42 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https:#www.datadoghq.com/).
+# Copyright 2016-present Datadog, Inc.
+
+require './lib/ostools.rb'
+
+name 'datadog-agent-python-runtime-lite'
+
+license "Apache-2.0"
+license_file "../LICENSE"
+
+# This software packages Python runtime and debugging tools for lite flavor
+dependency 'python3'
+dependency 'pympler'
+
+build do
+  # For lite flavor, create separate Python runtime package
+  if ENV['AGENT_FLAVOR'] == 'lite'
+    python_runtime_dir = "#{install_dir}/python-runtime"
+    mkdir python_runtime_dir
+
+    # Copy Python runtime to separate directory
+    if windows_target?
+      copy "#{install_dir}/embedded3/*", "#{python_runtime_dir}/"
+    else
+      copy "#{install_dir}/embedded/*", "#{python_runtime_dir}/"
+    end
+
+    # Create marker file to indicate this is a lite Python runtime package
+    command "echo 'lite' > #{python_runtime_dir}/AGENT_FLAVOR"
+    command "echo 'python-runtime' > #{python_runtime_dir}/COMPONENT_TYPE"
+
+    # Set proper permissions
+    if windows_target?
+      command "icacls #{windows_safe_path(python_runtime_dir)} /T /Q /C /RESET"
+    else
+      command "find #{python_runtime_dir} -type f -exec chmod 644 {} +"
+      command "find #{python_runtime_dir} -type d -exec chmod 755 {} +"
+    end
+  end
+end

omnibus/config/software/datadog-agent.rb

@@ -92,13 +92,27 @@
     end
     command "dda inv -- -e rtloader.clean", :live_stream => Omnibus.logger.live_stream(:info)
     command "dda inv -- -e rtloader.make --install-prefix \"#{windows_safe_path(python_3_embedded)}\" --cmake-options \"-G \\\"Unix Makefiles\\\" \\\"-DPython3_EXECUTABLE=#{windows_safe_path(python_3_embedded)}\\python.exe\\\" \\\"-DCMAKE_BUILD_TYPE=RelWithDebInfo\\\"\"", :env => env, :live_stream => Omnibus.logger.live_stream(:info)
-    command "mv rtloader/bin/*.dll  #{install_dir}/bin/agent/"
+
+    # For lite flavor, save Python components as separate artifacts
+    if ENV['AGENT_FLAVOR'] == 'lite'
+      mkdir "#{install_dir}/python-runtime"
+      copy "rtloader/bin/*", "#{install_dir}/python-runtime/"
+    else
+      command "mv rtloader/bin/*.dll  #{install_dir}/bin/agent/"
+    end
     command "dda inv -- -e agent.build --exclude-rtloader --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded #{do_windows_sysprobe} --flavor #{flavor_arg}", env: env, :live_stream => Omnibus.logger.live_stream(:info)
     command "dda inv -- -e systray.build", env: env, :live_stream => Omnibus.logger.live_stream(:info)
   else
     command "dda inv -- -e rtloader.clean", :live_stream => Omnibus.logger.live_stream(:info)
     command "dda inv -- -e rtloader.make --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER -DPython3_EXECUTABLE=#{install_dir}/embedded/bin/python3'", :env => env, :live_stream => Omnibus.logger.live_stream(:info)
-    command "dda inv -- -e rtloader.install", :live_stream => Omnibus.logger.live_stream(:info)
+
+    # For lite flavor, save Python runtime as separate artifact
+    if ENV['AGENT_FLAVOR'] == 'lite'
+      mkdir "#{install_dir}/python-runtime"
+      command "dda inv -- -e rtloader.install --install-prefix \"#{install_dir}/python-runtime\"", :live_stream => Omnibus.logger.live_stream(:info)
+    else
+      command "dda inv -- -e rtloader.install", :live_stream => Omnibus.logger.live_stream(:info)
+    end
 
     include_sds = ""
     if linux_target?
@@ -321,10 +335,21 @@
 
   block do
     python_scripts_dir = "#{project_dir}/omnibus/python-scripts"
-    mkdir "#{install_dir}/python-scripts"
-    Dir.glob("#{python_scripts_dir}/*").each do |file|
-      unless File.basename(file).end_with?('_tests.py')
-        copy file, "#{install_dir}/python-scripts"
+
+    # For lite flavor, put Python scripts with Python runtime artifacts
+    if ENV['AGENT_FLAVOR'] == 'lite'
+      mkdir "#{install_dir}/python-runtime/scripts"
+      Dir.glob("#{python_scripts_dir}/*").each do |file|
+        unless File.basename(file).end_with?('_tests.py')
+          copy file, "#{install_dir}/python-runtime/scripts"
+        end
+      end
+    else
+      mkdir "#{install_dir}/python-scripts"
+      Dir.glob("#{python_scripts_dir}/*").each do |file|
+        unless File.basename(file).end_with?('_tests.py')
+          copy file, "#{install_dir}/python-scripts"
+        end
       end
     end
   end

tasks/flavor.py

@@ -8,9 +8,13 @@ class AgentFlavor(enum.Enum):
     heroku = 3
     dogstatsd = 4
     fips = 5
+    lite = 6
 
     def is_iot(self):
         return self == type(self).iot
 
     def is_fips(self):
         return self == type(self).fips
+
+    def is_lite(self):
+        return self == type(self).lite