diff --git a/omnibus/config/projects/agent.rb b/omnibus/config/projects/agent.rb index cb5027f25b8bca..0bc3ab2f9426c4 100644 --- a/omnibus/config/projects/agent.rb +++ b/omnibus/config/projects/agent.rb @@ -267,6 +267,22 @@ # will package is the correct one disable_version_manifest do_package +# For lite flavor, create separate Python artifacts +if flavor == 'lite' && do_build + # Create Python runtime artifact + if Dir.exist?("#{install_dir}/python-runtime") + extra_package_file "#{install_dir}/python-runtime" => "datadog-lite-agent-python-runtime" + end + + # Create Python integrations artifact + if Dir.exist?("#{install_dir}/python-integrations") + extra_package_file "#{install_dir}/python-integrations" => "datadog-lite-agent-python-integrations" + end + + # Exclude Python artifacts from main package + exclude 'python-runtime' + exclude 'python-integrations' +end if linux_target? extra_package_file "#{output_config_dir}/etc/datadog-agent/" diff --git a/omnibus/config/software/datadog-agent-dependencies.rb b/omnibus/config/software/datadog-agent-dependencies.rb index aa5d26a6266094..41230ca170fc6f 100644 --- a/omnibus/config/software/datadog-agent-dependencies.rb +++ b/omnibus/config/software/datadog-agent-dependencies.rb @@ -22,7 +22,10 @@ dependency 'jmxfetch' # Used for memory profiling with the `status py` agent subcommand -dependency 'pympler' +# For lite flavor, pympler goes with Python runtime artifacts +unless ENV['AGENT_FLAVOR'] == 'lite' + dependency 'pympler' +end dependency "systemd" if linux_target? @@ -31,7 +34,13 @@ # Include traps db file in snmp.d/traps_db/ dependency 'snmp-traps' -dependency 'datadog-agent-integrations-py3' +# Build Python integrations but package separately for lite flavor +if ENV['AGENT_FLAVOR'] == 'lite' + dependency 'datadog-agent-integrations-py3-lite' + dependency 'datadog-agent-python-runtime-lite' +else + dependency 'datadog-agent-integrations-py3' +end # Additional software diff --git a/omnibus/config/software/datadog-agent-integrations-py3-lite.rb b/omnibus/config/software/datadog-agent-integrations-py3-lite.rb new file mode 100644 index 00000000000000..b1d105c9bd3004 --- /dev/null +++ b/omnibus/config/software/datadog-agent-integrations-py3-lite.rb @@ -0,0 +1,95 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https:#www.datadoghq.com/). +# Copyright 2016-present Datadog, Inc. + +require './lib/ostools.rb' +require 'json' + +name 'datadog-agent-integrations-py3-lite' + +license "BSD-3-Clause" +license_file "./LICENSE" + +dependency 'datadog-agent-integrations-py3-dependencies' + +python_version = "3.13" + +relative_path 'integrations-core' +whitelist_file "embedded/lib/python#{python_version}/site-packages/.libsaerospike" +whitelist_file "embedded/lib/python#{python_version}/site-packages/aerospike.libs" +whitelist_file "embedded/lib/python#{python_version}/site-packages/psycopg_binary.libs" +whitelist_file "embedded/lib/python#{python_version}/site-packages/pymqi" + +source git: 'https://github.com/DataDog/integrations-core.git' + +integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION'] +if integrations_core_version.nil? || integrations_core_version.empty? + integrations_core_version = 'master' +end +default_version integrations_core_version + +build do + # Set the PYTHONPATH and PYTHONHOME + env = { + "PYTHONPATH" => "#{install_dir}/embedded/lib/python#{python_version}/site-packages", + "PYTHONHOME" => "#{install_dir}/embedded", + } + + # For lite flavor, install integrations to separate directory + integrations_dir = "#{install_dir}/python-integrations" + mkdir integrations_dir + mkdir "#{integrations_dir}/lib/python#{python_version}/site-packages" + + env["PYTHONPATH"] = "#{integrations_dir}/lib/python#{python_version}/site-packages" + + pip_install_args = "#{windows_safe_path(install_dir)}\\embedded3\\python.exe -m pip install --no-deps --no-binary=:all:" if windows_target? + pip_install_args = "#{install_dir}/embedded/bin/pip3 install --no-deps --no-binary=:all:" unless windows_target? + + patch :source => 'create-regex-at-runtime.patch', :env => env if ohai['platform'] == "windows" && !arm64_target? + + # Get static requirements + env_script_path = "#{windows_safe_path(install_dir)}/embedded3/Scripts/Activate.ps1" + env_script_path = ". #{install_dir}/embedded/bin/activate" unless windows_target? + + requirements_file = 'omnibus/config/templates/datadog-agent-integrations-py3/static_requirements.txt' + + if windows_target? + requirements_file = 'omnibus\\config\\templates\\datadog-agent-integrations-py3\\static_requirements.txt' + end + + erb :dest => "#{Dir.pwd}/static_requirements.txt", + :source => "#{windows_safe_path(Dir.pwd)}/#{requirements_file}.erb", + :mode => 0755, + :vars => { :python_version => python_version, :requirements => {}, :static_reqs_in_requirements => [] } + + # Use the static requirements file to install base requirements to python-integrations dir + if windows_target? + install_cmd = "#{pip_install_args} --target #{integrations_dir}\\lib\\python#{python_version}\\site-packages" + else + install_cmd = "#{pip_install_args} --target #{integrations_dir}/lib/python#{python_version}/site-packages" + end + + command "#{install_cmd} -r static_requirements.txt", :env => env + + # Install integrations to separate directory + # Use smaller core set for lite flavor + core_integrations = %w[cpu disk io memory network uptime] + + core_integrations.each do |integration| + if File.exist?("#{integration}/setup.py") + command "#{install_cmd} ./#{integration}", :env => env, :cwd => "#{project_dir}" + end + end + + # Create marker file to indicate this is a lite integrations package + command "echo 'lite' > #{integrations_dir}/AGENT_FLAVOR" + + # Set proper permissions + if ohai['platform'] == "windows" + command "icacls #{windows_safe_path(integrations_dir)} /T /Q /C /RESET" + else + command "find #{integrations_dir} -type f -exec chmod 644 {} +" + command "find #{integrations_dir} -type d -exec chmod 755 {} +" + end +end \ No newline at end of file diff --git a/omnibus/config/software/datadog-agent-python-runtime-lite.rb b/omnibus/config/software/datadog-agent-python-runtime-lite.rb new file mode 100644 index 00000000000000..b29056a189ea3d --- /dev/null +++ b/omnibus/config/software/datadog-agent-python-runtime-lite.rb @@ -0,0 +1,42 @@ +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https:#www.datadoghq.com/). +# Copyright 2016-present Datadog, Inc. + +require './lib/ostools.rb' + +name 'datadog-agent-python-runtime-lite' + +license "Apache-2.0" +license_file "../LICENSE" + +# This software packages Python runtime and debugging tools for lite flavor +dependency 'python3' +dependency 'pympler' + +build do + # For lite flavor, create separate Python runtime package + if ENV['AGENT_FLAVOR'] == 'lite' + python_runtime_dir = "#{install_dir}/python-runtime" + mkdir python_runtime_dir + + # Copy Python runtime to separate directory + if windows_target? + copy "#{install_dir}/embedded3/*", "#{python_runtime_dir}/" + else + copy "#{install_dir}/embedded/*", "#{python_runtime_dir}/" + end + + # Create marker file to indicate this is a lite Python runtime package + command "echo 'lite' > #{python_runtime_dir}/AGENT_FLAVOR" + command "echo 'python-runtime' > #{python_runtime_dir}/COMPONENT_TYPE" + + # Set proper permissions + if windows_target? + command "icacls #{windows_safe_path(python_runtime_dir)} /T /Q /C /RESET" + else + command "find #{python_runtime_dir} -type f -exec chmod 644 {} +" + command "find #{python_runtime_dir} -type d -exec chmod 755 {} +" + end + end +end \ No newline at end of file diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 936134b4696c30..cc3722529b2aa0 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -92,13 +92,27 @@ end command "dda inv -- -e rtloader.clean", :live_stream => Omnibus.logger.live_stream(:info) command "dda inv -- -e rtloader.make --install-prefix \"#{windows_safe_path(python_3_embedded)}\" --cmake-options \"-G \\\"Unix Makefiles\\\" \\\"-DPython3_EXECUTABLE=#{windows_safe_path(python_3_embedded)}\\python.exe\\\" \\\"-DCMAKE_BUILD_TYPE=RelWithDebInfo\\\"\"", :env => env, :live_stream => Omnibus.logger.live_stream(:info) - command "mv rtloader/bin/*.dll #{install_dir}/bin/agent/" + + # For lite flavor, save Python components as separate artifacts + if ENV['AGENT_FLAVOR'] == 'lite' + mkdir "#{install_dir}/python-runtime" + copy "rtloader/bin/*", "#{install_dir}/python-runtime/" + else + command "mv rtloader/bin/*.dll #{install_dir}/bin/agent/" + end command "dda inv -- -e agent.build --exclude-rtloader --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded #{do_windows_sysprobe} --flavor #{flavor_arg}", env: env, :live_stream => Omnibus.logger.live_stream(:info) command "dda inv -- -e systray.build", env: env, :live_stream => Omnibus.logger.live_stream(:info) else command "dda inv -- -e rtloader.clean", :live_stream => Omnibus.logger.live_stream(:info) command "dda inv -- -e rtloader.make --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER -DPython3_EXECUTABLE=#{install_dir}/embedded/bin/python3'", :env => env, :live_stream => Omnibus.logger.live_stream(:info) - command "dda inv -- -e rtloader.install", :live_stream => Omnibus.logger.live_stream(:info) + + # For lite flavor, save Python runtime as separate artifact + if ENV['AGENT_FLAVOR'] == 'lite' + mkdir "#{install_dir}/python-runtime" + command "dda inv -- -e rtloader.install --install-prefix \"#{install_dir}/python-runtime\"", :live_stream => Omnibus.logger.live_stream(:info) + else + command "dda inv -- -e rtloader.install", :live_stream => Omnibus.logger.live_stream(:info) + end include_sds = "" if linux_target? @@ -321,10 +335,21 @@ block do python_scripts_dir = "#{project_dir}/omnibus/python-scripts" - mkdir "#{install_dir}/python-scripts" - Dir.glob("#{python_scripts_dir}/*").each do |file| - unless File.basename(file).end_with?('_tests.py') - copy file, "#{install_dir}/python-scripts" + + # For lite flavor, put Python scripts with Python runtime artifacts + if ENV['AGENT_FLAVOR'] == 'lite' + mkdir "#{install_dir}/python-runtime/scripts" + Dir.glob("#{python_scripts_dir}/*").each do |file| + unless File.basename(file).end_with?('_tests.py') + copy file, "#{install_dir}/python-runtime/scripts" + end + end + else + mkdir "#{install_dir}/python-scripts" + Dir.glob("#{python_scripts_dir}/*").each do |file| + unless File.basename(file).end_with?('_tests.py') + copy file, "#{install_dir}/python-scripts" + end end end end diff --git a/tasks/flavor.py b/tasks/flavor.py index 44d079a6e33164..371128d3312f2c 100644 --- a/tasks/flavor.py +++ b/tasks/flavor.py @@ -8,9 +8,13 @@ class AgentFlavor(enum.Enum): heroku = 3 dogstatsd = 4 fips = 5 + lite = 6 def is_iot(self): return self == type(self).iot def is_fips(self): return self == type(self).fips + + def is_lite(self): + return self == type(self).lite