From 5beb8906955d6e82d2c6514bc463b8426c6eac16 Mon Sep 17 00:00:00 2001 From: Adam Gross Date: Fri, 6 Nov 2020 14:36:10 -0500 Subject: [PATCH 1/3] [WIP] Proof-of-concept remote caching support This change introduces remote caching support to SCons. Currently SCons has support for caching built objects in directories, but remote caching adds support for sharing built objects across multiple machines. For example at VMware, we have our official builds push to cache and all developers will fetch from cache. The end result of this is that developers are able to only build what they have changed locally. All other cacheable build actions will be fetched from the remote cache server, dramatically speeding up builds. One piece that I introduced as part of this change is a new scheduler as a new class ParallelV2 that is used if remote caching is enabled or if a developer passes --use-scheduler-v2. Whereas the existing parallel scheduler waits on jobs if the job queue is full, this new scheduler will attempt to continue scanning for new jobs whenever possible. In addition, it supports remote caching by draining the "pending remote cache fetches" queue just like it drains the "pending jobs" queue. I believe that this scheduler is an alternative to https://github.com/SCons/scons/pull/3386 that should have the same effect in keeping jobs as high as possible. I introduced it as a separate class for now, but it could replace the Parallel class if we wanted. The new RemoteCache class owns the job of fetch from and pushing to a Bazel remote cache server or any other similar server that supports /ac/ and /cas/ GET and PUT requests using SHA-256 file names. See https://github.com/buchgr/bazel-remote for more details on the server. This class uses urllib3 for network requests. I chose it because it has good support for concurrency across threads using its ConnectionPool class. As part of implementing this functionality, the following new parameters are introduced: --remote-cache-fetch-enabled: Enables fetch of build output from the server --remote-cache-push-enabled: Enables push of build output to the server --remote-cache-url: Required if fetch or push is enabled --remote-cache-connections: Connection count (defaults to 100) --- .appveyor/install.bat | 2 +- CHANGES.txt | 14 + SCons/Job.py | 235 ++++- SCons/Node/FS.py | 23 +- SCons/Node/__init__.py | 23 +- SCons/RemoteCache.py | 949 ++++++++++++++++++ SCons/RemoteCacheTests.py | 358 +++++++ SCons/Script/Main.py | 43 +- SCons/Script/SConsOptions.py | 32 + SCons/Taskmaster.py | 89 +- bin/files | 1 + setup.cfg | 1 + test/RemoteCache/.exclude_tests | 2 + test/RemoteCache/CachePushAndFetch.py | 101 ++ test/RemoteCache/CachePushAndFetch/SConstruct | 2 + test/RemoteCache/RemoteCacheTestServer.py | 81 ++ test/RemoteCache/RemoteCacheUtils.py | 64 ++ 17 files changed, 1949 insertions(+), 71 deletions(-) create mode 100644 SCons/RemoteCache.py create mode 100644 SCons/RemoteCacheTests.py create mode 100644 test/RemoteCache/.exclude_tests create mode 100644 test/RemoteCache/CachePushAndFetch.py create mode 100644 test/RemoteCache/CachePushAndFetch/SConstruct create mode 100644 test/RemoteCache/RemoteCacheTestServer.py create mode 100644 test/RemoteCache/RemoteCacheUtils.py diff --git a/.appveyor/install.bat b/.appveyor/install.bat index d009e54d0f..a13ee5affd 100644 --- a/.appveyor/install.bat +++ b/.appveyor/install.bat @@ -3,7 +3,7 @@ for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys REM use mingw 32 bit until #3291 is resolved set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH% C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off coverage codecov +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off urllib3 coverage codecov set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements.txt REM install 3rd party tools to test with diff --git a/CHANGES.txt b/CHANGES.txt index fd316d3a80..ec36e4714e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,20 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER _concat function. If set to False, it will prepend and append $( and $). That way the various Environment variables can use that rather than "$( _concat(...) $)". + From Adam Gross: + - Added support for remote caching. This feature allows for fetch and push of build outputs + to a Bazel remote cache server or any other similar server that supports /ac/ and /cas/ + GET and PUT requests using SHA-256 file names. See https://github.com/buchgr/bazel-remote + for more details on the server. New parameters introduced: + --remote-cache-fetch-enabled: Enables fetch of build output from the server + --remote-cache-push-enabled: Enables push of build output to the server + --remote-cache-url: Required if fetch or push is enabled + --remote-cache-connections: Connection count (defaults to 100) + - Added support for a new parameter --use-scheduler-v2 that opts into a newer, more aggressive + parallel scanner. This scanner avoids waiting on jobs if the job queue is full and instead + scans for tasks. This scanner is expected to improve the performance of your build as long + as you don't have very large actions that cause poor scanning performance. + From David H: - Fix Issue #3906 - `IMPLICIT_COMMAND_DEPENDENCIES` was not properly disabled when set to any string value (For example ['none','false','no','off']) diff --git a/SCons/Job.py b/SCons/Job.py index f87a3bbfe6..b8d76b056d 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -21,14 +21,16 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""Serial and Parallel classes to execute build tasks. +"""Serial, Parallel, and ParallelV2 classes to execute build tasks. The Jobs class provides a higher level interface to start, stop, and wait on jobs. """ import SCons.compat +import SCons.Node +from collections import deque import os import signal @@ -64,7 +66,8 @@ class Jobs: methods for starting, stopping, and waiting on all N jobs. """ - def __init__(self, num, taskmaster): + def __init__(self, num, taskmaster, remote_cache=None, + use_scheduler_v2=False): """ Create 'num' jobs using the given taskmaster. @@ -76,19 +79,29 @@ def __init__(self, num, taskmaster): allocated. If more than one job is requested but the Parallel class can't do it, it gets reset to 1. Wrapping interfaces that care should check the value of 'num_jobs' after initialization. + + 'remote_cache' can be set to a RemoteCache.RemoteCache object. + + 'use_scheduler_v2' can be set to True to opt into the newer and more + aggressive scheduler. """ self.job = None - if num > 1: - stack_size = explicit_stack_size - if stack_size is None: - stack_size = default_stack_size - try: + stack_size = explicit_stack_size + if stack_size is None: + stack_size = default_stack_size + + try: + if ((remote_cache and remote_cache.fetch_enabled) or + use_scheduler_v2): + self.job = ParallelV2(taskmaster, num, stack_size, remote_cache) + elif num > 1: self.job = Parallel(taskmaster, num, stack_size) - self.num_jobs = num - except NameError: - pass + self.num_jobs = num + except NameError: + pass + if self.job is None: self.job = Serial(taskmaster) self.num_jobs = 1 @@ -359,7 +372,6 @@ def __init__(self, taskmaster, num, stack_size): self.taskmaster = taskmaster self.interrupted = InterruptState() self.tp = ThreadPool(num, stack_size, self.interrupted) - self.maxjobs = num def start(self): @@ -399,28 +411,199 @@ def start(self): # Let any/all completed tasks finish up before we go # back and put the next batch of tasks on the queue. while True: - task, ok = self.tp.get() + self.process_result() jobs = jobs - 1 - if ok: - task.executed() - else: - if self.interrupted(): - try: - raise SCons.Errors.BuildError( - task.targets[0], errstr=interrupt_msg) - except: - task.exception_set() - - # Let the failed() callback function arrange - # for the build to stop if that's appropriate. - task.failed() + if self.tp.resultsQueue.empty(): + break + + self.tp.cleanup() + self.taskmaster.cleanup() + + def process_result(self): + task, ok = self.tp.get() + + if ok: + task.executed() + else: + if self.interrupted(): + try: + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + except: + task.exception_set() + + # Let the failed() callback function arrange + # for the build to stop if that's appropriate. + task.failed() + + task.postprocess() + + class ParallelV2(Parallel): + """ + This class is an extension of the Parallel class that provides two main + improvements: + + 1. Minimizes time waiting for jobs by fetching tasks. + 2. Supports remote caching. + """ + __slots__ = ['remote_cache'] + + def __init__(self, taskmaster, num, stack_size, remote_cache): + super(ParallelV2, self).__init__(taskmaster, num, stack_size) + + self.remote_cache = remote_cache + def get_next_task_to_execute(self, limit): + """ + Finds the next task that is ready for execution. If limit is 0, + this function fetches until a task is found ready to execute. + Otherwise, this function will fetch up to "limit" number of tasks. + + Returns tuple with: + 1. Task to execute. + 2. False if a call to next_task returned None, True otherwise. + """ + count = 0 + while limit == 0 or count < limit: + task = self.taskmaster.next_task() + if task is None: + return None, False + + try: + # prepare task for execution + task.prepare() + except: + task.exception_set() + task.failed() task.postprocess() + else: + if task.needs_execute(): + return task, True + else: + task.executed() + task.postprocess() - if self.tp.resultsQueue.empty(): + count = count + 1 + + # We hit the limit of tasks to retrieve. + return None, True + + def start(self): + fetch_response_queue = queue.Queue(0) + if self.remote_cache: + self.remote_cache.set_fetch_response_queue( + fetch_response_queue) + + jobs = 0 + tasks_left = True + pending_fetches = 0 + cache_hits = 0 + cache_misses = 0 + cache_skips = 0 + cache_suspended = 0 + + while True: + fetch_limit = 0 if jobs == 0 and pending_fetches == 0 else 1 + if tasks_left: + task, tasks_left = \ + self.get_next_task_to_execute(fetch_limit) + else: + task = None + + if not task and not tasks_left and jobs == 0 and \ + pending_fetches == 0: + # No tasks left, no jobs, no cache fetches. + break + + while jobs > 0: + # Break if there are no results available and one of the + # following is true: + # 1. There are tasks left. + # 2. There is at least one job slot open and at least one + # remote cache fetch pending. + # Otherwise we want to wait for jobs because the most + # important factor for build speed is keeping the job + # queue full. + if ((tasks_left or + (jobs < self.maxjobs and pending_fetches > 0)) + and self.tp.resultsQueue.empty()): + break + + self.process_result() + jobs = jobs - 1 + + # Tasks could have been unblocked, so we should check + # again. + tasks_left = True + + while pending_fetches > 0: + # Trimming the remote cache fetch queue is the least + # important job, so we only block if there are no responses + # available, no tasks left to fetch, and no active jobs. + if ((tasks_left or jobs > 0) and + fetch_response_queue.empty()): break + cache_task, cache_hit, target_infos = \ + fetch_response_queue.get() + pending_fetches = pending_fetches - 1 + + if cache_hit: + cache_hits = cache_hits + 1 + cache_task.executed(target_infos=target_infos) + cache_task.postprocess() + + # Tasks could have been unblocked, so we should check + # again. + tasks_left = True + else: + cache_misses = cache_misses + 1 + self.tp.put(cache_task) + jobs = jobs + 1 + + if task: + # Tasks should first go to the remote cache if enabled. + if self.remote_cache: + fetch_pending, task_cacheable = \ + self.remote_cache.fetch_task(task) + else: + fetch_pending = task_cacheable = False + + if fetch_pending: + pending_fetches = pending_fetches + 1 + else: + # Fetch is not pending because remote cache is not + # being used or the task was not cacheable. + # + # Count the number of non-cacheable tasks but don't + # count tasks with 1 target that is an alias, because + # they are not actually run. + if (len(task.targets) > 1 or + not isinstance(task.targets[0], + SCons.Node.Alias.Alias)): + if task_cacheable: + cache_suspended = cache_suspended + 1 + else: + cache_skips = cache_skips + 1 + self.tp.put(task) + jobs = jobs + 1 + + # Instruct the remote caching layer to log information about + # the cache hit rate. + cache_count = cache_hits + cache_misses + cache_suspended + task_count = cache_count + cache_skips + if self.remote_cache and task_count > 0: + reset_count = self.remote_cache.reset_count + total_failures = self.remote_cache.total_failure_count + hit_pct = (cache_hits * 100.0 / cache_count if cache_count + else 0.0) + cacheable_pct = cache_count * 100.0 / task_count + self.remote_cache.log_stats( + hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, cacheable_pct, cache_skips, task_count, + total_failures, reset_count) + self.tp.cleanup() self.taskmaster.cleanup() diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a861bb..74f216e297 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -1118,6 +1118,9 @@ class LocalFS: really need this one? """ + def access(self, path, mode): + return os.access(path, mode) + def chmod(self, path, mode): return os.chmod(path, mode) @@ -3006,20 +3009,10 @@ def push_to_cache(self): if self.exists(): self.get_build_env().get_CacheDir().push(self) - def retrieve_from_cache(self): - """Try to retrieve the node's content from a cache - - This method is called from multiple threads in a parallel build, - so only do thread safe stuff here. Do thread unsafe stuff in - built(). - - Returns true if the node was successfully retrieved. + def should_retrieve_from_cache(self): + """Returns whether this node should be retrieved from the cache """ - if self.nocache: - return None - if not self.is_derived(): - return None - return self.get_build_env().get_CacheDir().retrieve(self) + return not self.nocache and self.is_derived() def visited(self): if self.exists() and self.executor is not None: @@ -3274,7 +3267,7 @@ def builder_set(self, builder): SCons.Node.Node.builder_set(self, builder) self.changed_since_last_build = 5 - def built(self): + def built(self, csig=None, size=0): """Called just after this File node is successfully built. Just like for 'release_target_info' we try to release @@ -3284,7 +3277,7 @@ def built(self): @see: release_target_info """ - SCons.Node.Node.built(self) + SCons.Node.Node.built(self, csig, size) if (not SCons.Node.interactive and not hasattr(self.attributes, 'keep_targetinfo')): diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ec742a686b..a27d99582f 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -681,6 +681,14 @@ def push_to_cache(self): """ pass + def should_retrieve_from_cache(self): + """Returns whether this node should be retrieved from the cache + + By default nodes are not cacheable. Child classes should override this + method if the CacheDir class supports them. + """ + return False + def retrieve_from_cache(self): """Try to retrieve the node's content from a cache @@ -690,7 +698,8 @@ def retrieve_from_cache(self): Returns true if the node was successfully retrieved. """ - return 0 + return (self.should_retrieve_from_cache() and + self.get_build_env().get_CacheDir().retrieve(self)) # # Taskmaster interface subsystem @@ -757,7 +766,7 @@ def build(self, **kw): e.node = self raise - def built(self): + def built(self, csig=None, size=0): """Called just after this node is successfully built.""" # Clear the implicit dependency caches of any Nodes @@ -793,7 +802,15 @@ def built(self): if not self.exists() and do_store_info: SCons.Warnings.warn(SCons.Warnings.TargetNotBuiltWarning, "Cannot find target " + str(self) + " after building") - self.ninfo.update(self) + + # If we already retrieved the NodeInfo from the cache, provide it now. + if csig: + self.ninfo = self.NodeInfo() + self.ninfo.update(self) + self.ninfo.csig = str(csig) + self.ninfo.size = size + else: + self.ninfo.update(self) def visited(self): """Called just after this node has been visited (with or diff --git a/SCons/RemoteCache.py b/SCons/RemoteCache.py new file mode 100644 index 0000000000..faee8e109b --- /dev/null +++ b/SCons/RemoteCache.py @@ -0,0 +1,949 @@ +"""SCons.RemoteCache + +This class owns the logic related to pushing to and fetching from a Bazel +remote cache server. That server stores Task metadata in the /ac/ section +(action cache) and Node binary data in the /cas/ section (content-addressible +storage). + +The Bazel remote cache uses LRU cache eviction for binaries in the +content-addressible storage. This means that Task metadata could exist in the +action cache /ac/ section but one or more of the Node binary data could be +missing due to eviction. Thus, in order for a Task to be fulfilled from cache, +we must verify that the Task metadata exists in the action cache and all Node +binary data exists in the content-addressible storage. + +This class can be configured to do cache fetch, cache push, or both. + +The first step of fetching a Task is to issue a GET request to the /ac/ +portion of the Bazel remote cache server. The server does validation of the +/ac/ request so it should only succeed if (1) the record exists on the server +and (2) the /cas/ storage has not evicted the nodes. + +We use two ThreadPoolExecutor instances. The first instance is responsible for +fetching a task from cache; a request to that executor is done when an entire +task fetch is completed, either resulting in a cache hit or cache miss. The +second instance is response for singular network accesses; a request to that +executor is done when the specified network request is completed. + +The second ThreadPoolExecutor is needed because the urllib3 API is synchronous +and we want multi-target tasks to be fetched in parallel. Requests to the first +ThreadPoolExecutor will wait on requests to the second ThreadPoolExecutor. If +we only used the one ThreadPoolExecutor and that executor was full of task +fetch requests, the cache would hang because there would be no room for the +individual network requests. We avoid that starvation by using the second +ThreadPoolExecutor. + +For security purposes and due to requirements by the Bazel remote cache server, +/ac/ and /cas/ requests always use SHA-256. This requires that SCons uses +SHA-256 content signatures. The security reasons are primarily to avoid +collisions with other actions. + +""" + +# +# Copyright 2020 VMware, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import SCons.Node.FS +import SCons.Script +import SCons.Util + +from concurrent.futures import ThreadPoolExecutor + +import atexit +import datetime +import json +import os +import queue +import random +import stat +import sys +import time +import threading + +try: + import urllib3 + urllib3_exception = None +except ImportError as e: + urllib3_exception = e + + +def raise_if_not_supported(): + # Verify that all required packages are present. + if urllib3_exception: + raise SCons.Errors.UserError( + 'Remote caching was requested but it is not supported in this ' + 'deployment. urllib3 is required but missing: %s' % + urllib3_exception) + + +class RemoteCache(object): + __slots__ = [ + 'backoff_remission_sec', + 'cache_ok_since', + 'connection_pool', + 'current_reset_backoff_multiplier', + 'debug_file', + 'failure_count', + 'fetch_enabled', + 'fetch_enabled_currently', + 'fetch_response_queue', + 'fs', + 'executor', + 'log', + 'metadata_request_timeout_sec', + 'metadata_version', + 'os_platform', + 'push_enabled', + 'push_enabled_currently', + 'request_executor', + 'request_failure_threshold', + 'reset_backoff_multiplier', + 'reset_count', + 'reset_delay_sec', + 'reset_skew_sec', + 'server_address', + 'server_path', + 'sessions', + 'stats_download_requests', + 'stats_download_total_bytes', + 'stats_download_total_ms', + 'stats_metadata_requests', + 'stats_metadata_total_ms', + 'stats_upload_requests', + 'stats_upload_total_bytes', + 'stats_upload_total_ms', + 'stats_lock', + 'total_failure_count', + 'transfer_request_timeout_sec' + ] + + def __init__(self, worker_count, server_address, fetch_enabled, + push_enabled, cache_debug): + """ + Initializes the cache. Supported parameters: + worker_count: Number of threads to create. + server_address: URL of remote server. Only the server name is + required, but the following can also be provided: + scheme, port, and url. + fetch_enabled: True if fetch is enabled. + push_enabled: True if push is enabled. + cache_debug: File to debug log to, '-' for stdout, None otherwise. + """ + self.debug_file = None + self.sessions = {} + self.fetch_enabled = fetch_enabled + self.push_enabled = push_enabled + self.log = SCons.Util.display + self.fs = SCons.Node.FS.default_fs # TODO: Use something better? + self.fetch_response_queue = None + self.failure_count = 0 + self.total_failure_count = 0 + self.cache_ok_since = 0.0 + self.reset_count = 0 + self.fetch_enabled_currently = self.fetch_enabled + self.push_enabled_currently = self.push_enabled + self.current_reset_backoff_multiplier = 1.0 + self.stats_lock = threading.Lock() + self.stats_download_requests = 0 + self.stats_download_total_bytes = 0 + self.stats_download_total_ms = 0 + self.stats_metadata_requests = 0 + self.stats_metadata_total_ms = 0 + self.stats_upload_requests = 0 + self.stats_upload_total_bytes = 0 + self.stats_upload_total_ms = 0 + + if cache_debug == '-': + self.debug_file = sys.stdout + elif cache_debug: + try: + self.debug_file = open(cache_debug, 'w') + except Exception as e: + self.log('WARNING: Unable to open cache debug file "%s" for ' + 'write with exception: %s.' % (cache_debug, e)) + + # This is hardcoded and expected to change if we ever change the + # contents of _get_task_metadata_signature. That allows us to + # continue to add more contents to the metadata over time and + # put it in a different place in the /ac/ section of the remote + # cache. + self.metadata_version = 1 + + # Include the platform in the signature. This is for the case where + # we have python actions that behave differently based on OS (so + # simply hashing the code isn't enough). + # + # Note: before python 3.3, 'linux2' was returned for linux kernels + # >2.6. Now it's just 'linux'. Other platforms (FreeBSD, etc) + # have also added version numbers. For our uses, we just strip + # them all. As a side effect, 'win32' becomes 'win'. + self.os_platform = sys.platform.rstrip('0123456789') + + # Maximum number of failures that we allow before disabling remote + # caching. + self.request_failure_threshold = 10 + + # Per-request timeouts. The first is for /ac/. The second is for /cas/. + self.metadata_request_timeout_sec = 10 + self.transfer_request_timeout_sec = 120 # 2 min + + # Time until we attempt a cache restart (+/- skew) + self.reset_delay_sec = 240.0 # 4 min + self.reset_skew_sec = 60.0 # 1 min + + # Exponential backoff multiplier + self.reset_backoff_multiplier = 1.5 + + # Time without errors before exponential backoff resets + self.backoff_remission_sec = 180.0 # 3 min + + # Generate the server address using the passed in data. + url_info = urllib3.util.parse_url(server_address) + if not url_info or not url_info.host: + raise SCons.Errors.UserError('An invalid remote cache server URL ' + '"%s" was provided' % server_address) + elif (url_info.scheme and + url_info.scheme.lower() not in ['http', 'https']): + raise SCons.Errors.UserError('Remote cache server URL must ' + 'start with http:// or https://') + self.server_address = '%s://%s' % ( + url_info.scheme if url_info.scheme else 'http', url_info.host) + if url_info.port: + self.server_address = '%s:%d' % (self.server_address, + url_info.port) + + # It is also allowable for the URL to contain a path, to which /ac/ and + # /cas/ requests should be appended. + self.server_path = url_info.path.rstrip('/') if url_info.path else '' + + # Create the thread pool and the connection pool that it uses. + # Have urllib3 perform 1 retry per request (its default is 3). 1 retry + # is helpful if in case of an unstable WiFi connection, while not + # taking too long to fail if the server is unreachable. + self.executor = ThreadPoolExecutor(max_workers=worker_count) + self.connection_pool = urllib3.connectionpool.connection_from_url( + self.server_address, maxsize=worker_count, block=True, + retries=1) + self.request_executor = ThreadPoolExecutor(max_workers=worker_count) + + self._debug('Remote cache server is configured as %s and max ' + 'connection count is %d.' % + (self.server_address + self.server_path, worker_count)) + + def _get_node_data_url(self, csig): + """Retrieves the URL for the specified node.""" + return '%s/cas/%s' % (self.server_path, csig) + + def _get_task_cache_url(self, task): + """Retrieves the URL for the specified task's metadata.""" + return '%s/ac/%s' % (self.server_path, + self._get_task_metadata_signature(task)) + + def _get_task_metadata_signature(self, task): + """ + Retrieves the SHA-256 signature that represents the task in the remote + cache. This is used to look up task metadata in the remote cache. + + Important note 1: if you ever change the output of _get_task_metadata, + you must also change metadata_version. + + Important note 2: When SCons supports SHA1 for content signatures, the + "alternative-hash-md5" string will need to be dynamic. + """ + sig_info = [ + 'scons-metadata-version-%d' % self.metadata_version, + 'os-platform=%s' % self.os_platform, + ] + [t.get_cachedir_bsig() for t in task.targets] + return SCons.Util.hash_signature(';'.join(sig_info)) + + def _get_task_metadata(self, task): + """ + Retrieves the JSON metadata that we want to push to the action cache, + dumped to a string and encoded using UTF-8. + + As we are using the Bazel remote cache server, which validates the + entries placed into /ac/, this must match the JSON encoding of Bazel's + ActionResult protobuf message from remote_execution.proto. + """ + output_files = [] + is_posix = os.name == 'posix' + for t in task.targets: + output_file = { + 'path': t.path, + 'digest': { + 'hash': t.get_csig(), + 'sizeBytes': t.get_size(), + }, + } + if is_posix: + output_file['isExecutable'] = t.fs.access(t.abspath, os.X_OK) + output_files += [output_file] + return json.dumps({'outputFiles': output_files}).encode('utf-8') + + def set_fetch_response_queue(self, queue): + """ + Sets the queue used to report cache fetch results if fetching + is enabled. + """ + self.fetch_response_queue = queue + + def fetch_task(self, task): + """ + Dispatches a request to a helper thread to fetch a task from the + remote cache. + + Returns tuple with two booleans: + [0]: True if we submitted the task to the thread pool and False + in all other cases. + [1]: True if the task is cacheable *and* caching was enabled at + some point in the build. False otherwise. + """ + if not (self.fetch_enabled and task.is_cacheable()): + # Caching is not enabled in general or for this task. + return False, False + elif (not self.fetch_enabled_currently and + not self._try_reset_failure()): + # The cache is currently disabled. + return False, True + else: + self.executor.submit(self._fetch, task) + return True, True + + def push_task(self, task): + """ + Dispatches a request to a helper thread to push a task to the + remote cache. + """ + if not self.push_enabled or not task.is_cacheable(): + return + + if self.push_enabled_currently or self._try_reset_failure(): + self.executor.submit(self._push, task) + + def _request_nodes(self, node_tuples): + """ + Makes a GET request to the server for each of the specific Node content + signatures. + + Returns True if all requests succeeded, False otherwise. + """ + if not node_tuples: + return True + + if len(node_tuples) == 1: + # Optimize this common case by not going to the ThreadPoolExecutor. + # That would just cause unnecessary delays due to thread scheduling + # and waiting for locks. + return self._request(node_tuples[0], None) + else: + # We need to make more than one request and doing it serially would + # cause unnecessary delays for high-latency connections. urllib3 is + # a synchronous API so we work around that by doing the requests + # in a helper thread pool. + responses_left = len(node_tuples) + all_files_present = True + result_queue = queue.Queue(0) + + for node_tuple in node_tuples: + self.request_executor.submit(self._request, node_tuple, + result_queue) + + while responses_left > 0: + success = result_queue.get() + all_files_present = \ + all_files_present and success + responses_left -= 1 + + return all_files_present + + def _track_request(self, verb, url, target_path, *args, **kwargs): + """ + Wraps a GET request, tracking response time and optionally logging + details of the response. + + Supported parameters: + verb (String): Verb to request (PUT or GET). + url (String): URL to make the request to. + target_path (String): None if it's a metadata request. Otherwise, + the relative or absolute path for the target + we are putting/getting. + args/kwargs: Params to be passed into connection_pool.request. + """ + start = datetime.datetime.now() + is_metadata_request = target_path is None + + response = None + try: + if verb == 'GET': + response = self.connection_pool.request(verb, url, *args, + **kwargs) + else: + response = self.connection_pool.urlopen(verb, url, *args, + **kwargs) + exception = None + except Exception as e: + exception = e + + if self.debug_file: + with self.stats_lock: + ms = self._get_delta_ms(start) + if is_metadata_request: + self.stats_metadata_requests += 1 + self.stats_metadata_total_ms += ms + elif response is not None and self._success(response): + # /cas/ requests only provide good tracking data if they + # succeeded. Otherwise, the actual uploaded/downloaded size + # is unknown and would skew our averages. + if verb == 'GET': + self.stats_download_requests += 1 + self.stats_download_total_ms += ms + self.stats_download_total_bytes += len(response.data) + else: + self.stats_upload_requests += 1 + self.stats_upload_total_ms += ms + self.stats_upload_total_bytes += len(kwargs['body']) + + ms = self._get_delta_ms(start) + # target_path could be relative or absolute. Convert to relative. + target_log = (('for target %s ' % self.fs.File(target_path).path) + if target_path else '') + + if exception: + self._debug( + '%s FAILED exception %s: URL %s %s(%d ms elapsed).' % + (verb, exception, url, target_log, ms)) + else: + if self._success(response): + self._debug('%s SUCCESS: URL %s %s(%d ms elapsed).' % + (verb, url, target_log, ms)) + else: + self._debug( + '%s FAILED %d (%s): URL %s %s(%d ms elapsed).' % + (verb, response.status, response.reason, url, + target_log, ms)) + + if exception: + raise exception + + return response + + def _validate_file(self, request_data, csig, size, path): + """Validates that the downloaded file data is correct + + This function verifies that request_data matches the expected size in + bytes and SHA-256 content signature. + + Returns True if it was successfully validated, False otherwise. + """ + actual_size = len(request_data) + if actual_size != size: + # Validate the size of the downloaded data. + self.log('Size mismatch downloading file "%s". Expected size %d, ' + 'got %d.' % (path, size, actual_size)) + return False + + # Validate the hash of the downloaded data. We only use SHA-256 in + # so we only need to check for that. + csig_length = len(csig) + if csig_length == 64: + actual_csig = SCons.Util.hash_signature(request_data) + else: + self.log('WARNING: Not validating csig %s (path "%s") because ' + 'hash length %d is of an unknown format.' % + (csig, path, csig_length)) + actual_csig = None + + if actual_csig is not None and actual_csig != csig: + self.log('Hash mismatch downloading file "%s". Expected hash ' + '%s, but got %s.' % (path, csig, actual_csig)) + return False + + return True + + def _request(self, node_tuple, result_queue): + """ + Implementation of sending a GET request to the remote cache for a + specific node. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + + Params: + node_tuple (tuple): Tuple of node absolute path, csig, size, and + is_executable. is_executable is only used for + GET requests on Posix systems. + result_queue (Queue): Optional queue to post Boolean result to. + + Returns True if the request succeeded, False otherwise. + """ + (path, csig, size, is_executable) = node_tuple + + try: + # XXX TODO: Stream responses to GET requests by passing + # preload_content=False. + response = self._track_request( + 'GET', self._get_node_data_url(csig), path, + timeout=self.transfer_request_timeout_sec) + + if self._success(response): + # Validate that the data has the correct signature and size. + success = self._validate_file(response.data, csig, size, path) + if success: + with open(path, 'wb') as f: + f.write(response.data) + + if os.name == 'posix' and is_executable: + self.fs.chmod( + path, self.fs.stat(path).st_mode | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + if result_queue: + result_queue.put(success) + return success + except urllib3.exceptions.HTTPError as e: + # Host is not available. Immediately disable caching. + self.log('GET request failed for file %s with exception: %s' % + (path, e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception making GET request for file %s: ' + '%s, %s' % (path, e, traceback.format_exc())) + self._handle_failure() + + if result_queue: + result_queue.put(False) + return False + + def _success(self, response): + """Wraps some logic to determine whether the request succeeded.""" + return response.status < 400 or response.status >= 600 + + def _get_nodes_from_task_metadata(self, task_metadata): + """Retrieves node tuples from the task metadata. + + Returns a list of tuples with the following entries: + 1. Absolute path to the file. + 2. File content signature. + 3. Size in bytes. + 4. True if the file should be marked as executable on Posix, False + otherwise. + """ + nodes = [] + for output_file in task_metadata['outputFiles']: + path = output_file.get('path', None) + if path: + path = self.fs.File(path).abspath + digest = output_file.get('digest', None) + csig = digest.get('hash', None) + size = int(digest.get('sizeBytes', '0')) + is_executable = output_file.get('isExecutable', False) + + if path and digest and csig: + nodes.append((path, csig, size, is_executable)) + return nodes + + def _get_delta_ms(self, start): + """ + Helper function to return the difference in milliseconds between now + and the specified start time. + """ + delta = datetime.datetime.now() - start + return delta.total_seconds() * 1000 + + def _fetch(self, task): + """ + Implementation of fetching a task from a remote cache. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + """ + fetched_all_nodes = False + + try: + url = self._get_task_cache_url(task) + + # Fetch the task metadata from the server, if it exists. + # Catch JSON decoding errors in case we received a partial + # response from the server or the stored cache data is incomplete. + response = self._track_request( + 'GET', url, None, + headers={'Accept': 'application/json'}, + timeout=self.metadata_request_timeout_sec) + action_result = None + if self._success(response): + try: + action_result = json.loads(response.data.decode('utf-8')) + except json.JSONDecodeError as e: + if self.debug_file: + self._debug('Cache miss due to bad JSON data in the ' + 'action cache for task with url %s, ' + 'targets %s. Exception: %s' % + (url, [str(t) for t in task.targets], e)) + except Exception as e: + self.log('Received unexpected exception %s when trying to ' + 'decode JSON data for task with url %s, targets ' + '%s' % (e, url, [str(t) for t in task.targets])) + + if action_result: + nodes = self._get_nodes_from_task_metadata(action_result) + + if not nodes: + # The task metadata couldn't be processed. + if self.debug_file: + self._debug('Cache miss for task with url %s, targets ' + '%s due to bad /ac/ record.' % + (url, [str(t) for t in task.targets])) + elif self._request_nodes(nodes): + # All downloads were successful. The main python thread + # marks nodes as built, so we must deliver csig and size + # info in a format expected by the + # Task.executed_with_callbacks function. + target_infos = [(csig, size) for _, csig, size, _ in nodes] + + # Mark each target as cached so it isn't unnecessarily + # pushed back to cache. + for t in task.targets: + t.cached = 1 + + # Optionally print information about the cache hit. + if self.debug_file: + self._debug_cache_result(True, task, url) + + # Optionally push the result to the response queue. + if self.fetch_response_queue: + self.fetch_response_queue.put((task, True, + target_infos)) + return True + elif self.debug_file: + self._debug('Cache miss for task with url %s, targets ' + '%s due to missing /cas/ record.' % + (url, [str(t) for t in task.targets])) + elif self.debug_file: + self._debug_cache_result(False, task, url) + except urllib3.exceptions.HTTPError as e: + # Host could not be reached. + self.log('GET request failed for task (targets=%s) with ' + 'exception: %s' % ([str(t) for t in task.targets], e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception fetching task (targets=%s): ' + '%s, %s' % + ([str(t) for t in task.targets], e, + traceback.format_exc())) + self._handle_failure() + + # This is the failure path. Unlink any retrieved files and indicate + # failure. + for t in task.targets: + if t.fs.exists(t.abspath): + t.fs.unlink(t.abspath) + + # Push the result to the response queue. + if self.fetch_response_queue: + self.fetch_response_queue.put((task, False, None)) + else: + self.log('Unexpectedly unable to find a response queue to ' + 'push the remote cache fetch result to.') + + def _push(self, task): + """ + Implementation of pushing a task to a remote cache. + + This function can be called multiple times from different threads + in the ThreadPoolExecutor, so it must be thread-safe. + + :param task: Task instance to push. + """ + try: + # Push each node first then push the task info. + for t in task.targets: + response = self._track_request( + 'PUT', self._get_node_data_url(t.get_csig()), t.path, + body=t.get_contents(), + timeout=self.transfer_request_timeout_sec) + + if not self._success(response): + # If the write failed, don't continue, because we don't + # want to write the metadata. + return + + # All target writes succeeded, so write the metadata. + response = self._track_request( + 'PUT', self._get_task_cache_url(task), None, + body=self._get_task_metadata(task), + headers={'Content-Type': 'application/json'}, + timeout=self.metadata_request_timeout_sec) + + except urllib3.exceptions.HTTPError as e: + # Host is not available. + self.log('PUT request failed for task (targets=%s) with ' + 'exception: %s' % ([str(t) for t in task.targets], e)) + self._handle_failure() + except Exception as e: + import traceback + self.log('Unexpected exception pushing task (targets=%s): ' + '%s, %s' % + ([str(t) for t in task.targets], e, + traceback.format_exc())) + self._handle_failure() + + def close(self): + """Releases any resources that this class acquired.""" + if self.executor is not None: + # Async fetches shouldn't still be pending at this point, but + # async pushes could. + start = datetime.datetime.now() + self.executor.shutdown(wait=True) + if self.debug_file: + # Log a debug message if shutting down the network request + # executor took a while. This happens if there are large + # cache pushes at the end of the build. + ms = self._get_delta_ms(start) + if ms > 100: + self._debug('Shutting down took %d ms.' % ms) + + self.executor = None + + for _, session in self.sessions.items(): + session.close() + self.sessions.clear() + + if self.debug_file not in [None, sys.stdout]: + self.debug_file.close() + self.debug_file = None + + def _get_monotonic_now(self): + """ """ + if sys.version_info[:2] >= (3, 3): + return time.monotonic() + else: + return time.clock() + + def _handle_failure(self): + """Disables remote caching if there were too many failures.""" + self.failure_count += 1 + self.total_failure_count += 1 + + now = self._get_monotonic_now() + cache_ok_since = self.cache_ok_since + + if (self.failure_count > self.request_failure_threshold and + (self.push_enabled_currently or + self.fetch_enabled_currently)): + + cache_ok_duration = now - cache_ok_since + + # Note: backoff_remission_sec above is not scaled by the + # multiplier today. We may revisit that design choice later. + if (self.reset_count and + (not self.backoff_remission_sec or + cache_ok_duration < self.backoff_remission_sec)): + # This is an exponential backoff scenario. + backoff_multiplier = ( + self.current_reset_backoff_multiplier * + self.reset_backoff_multiplier) + else: + # Either this is the first failure or the enough time + # passed to reset the exponential backoff. + backoff_multiplier = 1.0 + + reset_text = 'RemoteCache: Request failure threshold was reached.' + + if self.reset_delay_sec: + # If the server is down, we don't want everybody trying to + # hit it all at once, so add some randomness. + # random.randint() requires that reset_skew_sec be an int. + reset_skew_sec = int(self.reset_skew_sec * backoff_multiplier) + reset_delay_sec = self.reset_delay_sec * backoff_multiplier + + skew = random.randint(-reset_skew_sec, reset_skew_sec) + delay = reset_delay_sec + skew + + # Set cache_ok_since into the future. + self.cache_ok_since = now + delay + + reset_text += (' Suspending remote caching, attempting ' + 'restart in %2.1f seconds.' % delay) + else: + reset_text += ' Disabling remote caching.' + + self.log(reset_text) + self.current_reset_backoff_multiplier = backoff_multiplier + self.push_enabled_currently = False + self.fetch_enabled_currently = False + elif cache_ok_since < now: + # The cache has behaved unhealthy up until this point. + # Note: cache_ok_since points into the future when the cache is + # disabled and we're waiting for a reset. + self.cache_ok_since = now + + def _try_reset_failure(self): + """ + Attempts to restart the cache fetch and push logic if the appropriate + amount of time has passed. Exponenial backoff is implemented as well. + :return: False is the reset deadline hasn't passed yet. + """ + if not self.reset_delay_sec: + return False + + now = self._get_monotonic_now() + + # When the cache is suspended, cache_ok_since is set into the future. + # once we reach that point, we re-enable it. + if now < self.cache_ok_since: + return False + + self.log('RemoteCache: Resuming remote caching attempts.') + self.reset_count += 1 + self.failure_count = 0 + self.fetch_enabled_currently = self.fetch_enabled + self.push_enabled_currently = self.push_enabled + return True + + def _debug(self, msg): + """ + Prints a debug message if --cache-debug was set. + Caller is responsible for checking that self.debug_file is not None. + """ + if self.debug_file: + self.debug_file.write(msg + '\n') + + def _debug_cache_result(self, hit, task, url): + """ + Prints the result of a task lookup to debug_file. + Caller is responsible for checking that self.debug_file is not None. + """ + t = task.targets[0] + executor = t.get_executor() + try: + # Print the raw executor contents without any subst calls. This + # makes it easier to compare hit and miss records across builds. + # Avoid using executor.get_contents() because it uses ''.join() + # and doesn't print well. + actions = [action.get_contents(executor.get_all_targets(), + executor.get_all_sources(), + executor.get_build_env()).decode() + for action in executor.get_action_list()] + except Exception: + # Actions that run Python functions fail because they can't + # be printed raw. Instead, convert directly to string. + actions = [str(executor)] + + def format_container(container): + """ + Returns the contents of the container, preserving space by putting + consecutive entries from the same directory on the same line. It is + extremely important that order is maintained in the outputted list, + so this function cannot do any sorting. This also means that the + same directory may show up multiple times in the returned string + if all files from that directory are not adjacent in the + container. + """ + lastdir = None + result = '' + for c in container: + dir = getattr(c, 'dir', None) + name = getattr(c, 'name', '') + # Note: this code uses concatenation instead of % formatting + # for better performance. + if dir is None: + result += '\n\t' + str(c) + elif dir != lastdir: + result += '\n\t' + str(dir) + ': ' + str(name) + else: + result += ', ' + str(name) + lastdir = dir + return result if result else ' None' + + # To be cacheable, there needs to be at least one file target, which + # means that we expect sources, depends, and implicit to be set. + # However, ignore may not be set. + self._debug('Cache %s for task with url %s.\nTargets:%s\n' + 'Sources:%s\nDepends:%s\nImplicit:%s\nIgnore:%s\n' + 'Actions:\n\t%s' % + ('hit' if hit else 'miss', url, + format_container(task.targets), + format_container(t.sources), + format_container(t.depends), + format_container(t.implicit), + format_container(getattr(t, 'ignore', [])), + '\n\t'.join(actions))) + + def _print_operation_stats(self, operation, requests, bytes, ms): + """Prints statistics about the specified operation.""" + if requests > 0: + mb = float(bytes) / 1048576 + seconds = float(ms) / 1000 + self._debug('%s stats: %d requests on %d MB took %d ms. Average ' + '%2.2f ms per MB and %2.2f MB per second.' % + (operation, requests, mb, ms, + 0 if mb == 0 else float(ms) / mb, + 0 if seconds == 0 else mb / seconds)) + + def log_stats(self, hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, cacheable_pct, cache_skips, task_count, + total_failures, reset_count): + """ + Prints the statistics that were collected during a build. + """ + message = ( + '%2.1f percent cache hit rate on %d cacheable tasks with %d hits, ' + '%d misses, %d w/cache suspended. %2.1f percent of total tasks ' + 'cacheable, due to %d/%d tasks marked not cacheable. Saw %d total ' + 'failures, %d cache restarts.' % + (hit_pct, cache_count, cache_hits, cache_misses, + cache_suspended, + cacheable_pct, cache_skips, task_count, + total_failures, reset_count) + ) + self.log('RemoteCache: %s' % message) + + if self.debug_file: + self._debug(message) + + if self.stats_metadata_requests > 0: + self._debug('Task metadata stats: %d ms average RTT on %d ' + 'requests.' % + (self.stats_metadata_total_ms / + self.stats_metadata_requests, + self.stats_metadata_requests)) + + # The upload and download stats use the same log format. + self._print_operation_stats('Download', + self.stats_download_requests, + self.stats_download_total_bytes, + self.stats_download_total_ms) + self._print_operation_stats('Upload', + self.stats_upload_requests, + self.stats_upload_total_bytes, + self.stats_upload_total_ms) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/RemoteCacheTests.py b/SCons/RemoteCacheTests.py new file mode 100644 index 0000000000..4c3505fc36 --- /dev/null +++ b/SCons/RemoteCacheTests.py @@ -0,0 +1,358 @@ +# MIT License +# +# Copyright 2020 VMware, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +from hashlib import sha256 +import json +import os +import queue +import stat +import sys +import unittest +from time import sleep +from unittest import mock + +from TestCmd import TestCmd +import TestSCons +import TestUnit + +import SCons.RemoteCache + +# If the Python version running the tests doesn't have urllib3, then +# RemoteCache.py will not have the urllib3 attribute. However, the various +# mock.patch decorators below depend upon the attribute existing. +if not hasattr(SCons.RemoteCache, 'urllib3'): + SCons.RemoteCache.urllib3 = None + + +class Url(): + """Test version of the urllib3's Url class.""" + __slots__ = ['host', 'scheme', 'port', 'path'] + + def __init__(self): + self.host = None + self.scheme = None + self.port = 0 + self.path = None + + +def MockParseUrl(address): + """Naive implementation of URL parsing. Just enough for tests.""" + url = Url() + components = address.split('://', 1) + if len(components) > 1: + if components[0]: + url.scheme = components[0] + else: + # Invalid URL starting with :// + return None + + components = components[-1].split('/', 1) + port_components = components[0].split(':', 1) + url.host = port_components[0] + if len(port_components) > 1: + url.port = int(port_components[1]) + else: + url.port = 443 if url.scheme == 'https' else 80 + if len(components) > 1: + url.path = components[1] + return url + + +class MockResponse(mock.MagicMock): + """Mock version of a response class that comes from urllib3.response.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, val in kwargs.items(): + setattr(self, key, val) + +class MockConnectionPool(mock.MagicMock): + """Mock version of the urllib3.ConnectionPool class.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.pending_responses = [] + + def add_pending_response(self, **kwargs): + self.pending_responses.append(MockResponse(**kwargs)) + + def request(self, verb, url, *args, **kwargs): + return self.pending_responses.pop(0) + + def urlopen(self, verb, url, *args, **kwargs): + return self.pending_responses.pop(0) + + +def MockConnectionFromUrl(*args, **kwargs): + """ + Mock side effect for function urllib3.ConnectionPool.connection_from_url. + Returns a mock version of urllib3.ConnectionPool. + """ + return MockConnectionPool(*args, **kwargs) + + +class Task(): + """Test version of the Task class.""" + slots = ['targets', 'cacheable'] + + def __init__(self, targets, cacheable): + self.targets = targets + self.cacheable = cacheable + + def is_cacheable(self): + return self.cacheable + + +def CreateRemoteCache(mock_urllib3, worker_count, server_address, + fetch_enabled, push_enabled): + """ + Creates an instance of the RemoteCache class using various mock classes. + """ + # Initialize all mocks. + mock_urllib3.util.parse_url = mock.MagicMock(side_effect=MockParseUrl) + mock_urllib3.connectionpool.connection_from_url = mock.MagicMock( + side_effect=MockConnectionFromUrl) + + # Create the remote cache and queue. + cache = SCons.RemoteCache.RemoteCache( + worker_count, server_address, fetch_enabled, push_enabled, None) + mock_urllib3.util.parse_url.assert_called_with(server_address) + assert cache != None, cache + q = queue.Queue(0) + cache.set_fetch_response_queue(q) + + return cache, q + + +def GetJSONMetadata(files): + """ + Retrieves the JSON metadata for the specified files. Files can either be a + single tuple or a list of tuples. Each tuple should have the following: + + 1. Filename. + 2. Contents. + 3. Hash. + 4. (Optional) True if the file is executable on posix. Default is False. + """ + if isinstance(files, tuple): + files = [files] + + output_files = [] + for file in files: + if len(file) == 4: + (filename, contents, hash, isExecutable) = file + else: + (filename, contents, hash) = file + isExecutable = False + + output_file = { + 'path': filename, + 'digest': { + 'hash': hash, + 'sizeBytes': len(contents), + }, + } + if os.name == 'posix': + output_file['isExecutable'] = isExecutable + output_files.append(output_file) + + return json.dumps({'outputFiles': output_files}).encode() + + +class TestCaseBase(unittest.TestCase): + """Base test case that does common test setup.""" + def setUp(self): + SCons.Util.set_hash_format('sha256') + self.test = TestSCons.TestSCons() + self.env = self.test.Environment() + self.name = self.__class__.__name__ + + +class FetchOneTargetTestCase(TestCaseBase): + """Fetches one task from cache.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + # Test making a request. + contents = b'a' + hash = sha256(contents).hexdigest() + data = GetJSONMetadata((self.name, contents, hash)) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=200, data=contents) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert t.cached + assert t.get_contents() == contents, t.get_contents() + assert t.get_csig() == hash, t.get_csig() + assert hit + assert target_infos == [(hash, len(contents))] + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class FetchMultipleTargetsTestCase(TestCaseBase): + """ + Tests fetching a task with multiple targets. This will result in /ac/ fetch + and multiple /cas/ fetches. + """ + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + # Test making a request. + filename1 = self.name + contents1 = b'a' + filename2 = filename1 + '2' + contents2 = b'b' + hash1 = sha256(contents1).hexdigest() + hash2 = sha256(contents2).hexdigest() + data = GetJSONMetadata([(filename1, contents1, hash1), + (filename2, contents2, hash2)]) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=200, data=contents1) + cache.connection_pool.add_pending_response(status=200, data=contents2) + + t1 = self.env.File(filename1) + t2 = self.env.File(filename2) + task = Task([t1, t2], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert t1.cached + assert t1.get_contents() == contents1, t1.get_contents() + assert t1.get_csig() == hash1, t1.get_csig() + assert t2.cached + assert t2.get_contents() == contents2, t2.get_contents() + assert t2.get_csig() == hash2, t2.get_csig() + assert hit + assert target_infos == [(hash1, len(contents1)), + (hash2, len(contents2))] + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class PushMultipleTargetsTestCase(TestCaseBase): + """ + Tests pushing a task with multiple targets. This will result in /cas/ + pushes and one /ac/ push. + """ + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', False, True) + + # Push two targets. If it's a posix machine, mark one of them as + # executable. + targets = [] + file_infos = [(self.name, b'a', sha256(b'a').hexdigest(), True), + (self.name + '2', b'b', sha256(b'b').hexdigest(), False)] + for name, contents, _, executable in file_infos: + t = self.env.File(name) + with open(t.abspath, 'w') as f: + f.write(contents.decode()) + + if os.name == 'posix' and executable: + self.env.fs.chmod( + t.abspath, + stat.S_IXUSR | self.env.fs.stat(t.abspath)[stat.ST_MODE]) + + targets.append(t) + + data = GetJSONMetadata(file_infos) + cache.connection_pool.add_pending_response(status=200, data=file_infos[0][1]) + cache.connection_pool.add_pending_response(status=200, data=file_infos[1][1]) + cache.connection_pool.add_pending_response(status=200, data=data) + + task = Task(targets, True) + cache.push_task(task) + connection_pool = cache.connection_pool + cache.close() # This will wait for ThreadPool requests to complete. + assert not connection_pool.pending_responses, \ + connection_pool.pending_responses + + +class FetchMetadataFailureTestCase(TestCaseBase): + """Tests a failed /ac/ request.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + cache.connection_pool.add_pending_response(status=404) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert not t.cached + assert not hit + assert target_infos == None + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +class FetchNodeContentsFailureTestCase(TestCaseBase): + """Tests a successful /ac/ request but failed /cas/ request.""" + @mock.patch('SCons.RemoteCache.urllib3') + def runTest(self, mock_urllib3): + cache, q = CreateRemoteCache(mock_urllib3, 4, 'test', True, False) + + contents = b'a' + hash = sha256(contents).hexdigest() + data = GetJSONMetadata((self.name, contents, hash)) + cache.connection_pool.add_pending_response(status=200, data=data) + cache.connection_pool.add_pending_response(status=404) + + t = self.env.File(self.name) + task = Task([t], True) + fetch_result = cache.fetch_task(task) + assert fetch_result == (True, True), fetch_result + + task2, hit, target_infos = q.get() + assert task == task2 + assert not t.cached + assert not hit + assert target_infos == None + assert not cache.connection_pool.pending_responses, \ + cache.connection_pool.pending_responses + + +if __name__ == "__main__": + unittest.main() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index b67469256d..c96c648876 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -57,6 +57,7 @@ import SCons.Node.FS import SCons.Platform import SCons.Platform.virtualenv +import SCons.RemoteCache import SCons.SConf import SCons.Script import SCons.Taskmaster @@ -226,7 +227,7 @@ def do_failed(self, status=2): exit_status = status this_build_status = status - def executed(self): + def executed(self, target_infos=None): t = self.targets[0] if self.top and not t.has_builder() and not t.side_effect: if not t.exists(): @@ -247,9 +248,11 @@ def executed(self): self.do_failed() else: print("scons: Nothing to be done for `%s'." % t) - SCons.Taskmaster.OutOfDateTask.executed(self) + SCons.Taskmaster.OutOfDateTask.executed( + self, target_infos=target_infos) else: - SCons.Taskmaster.OutOfDateTask.executed(self) + SCons.Taskmaster.OutOfDateTask.executed( + self, target_infos=target_infos) def failed(self): # Handle the failure of a build task. The primary purpose here @@ -423,7 +426,7 @@ def execute(self): this_build_status = 1 self.tm.stop() - def executed(self): + def executed(self, target_infos=None): pass @@ -1118,7 +1121,12 @@ def _main(parser): # Hash format and chunksize are set late to support SetOption being called # in a SConscript or SConstruct file. - SCons.Util.set_hash_format(options.hash_format) + # Remote caching requires SHA-256. + if (options.remote_cache_fetch_enabled or + options.remote_cache_push_enabled): + SCons.Util.set_hash_format('sha256') + else: + SCons.Util.set_hash_format(options.hash_format) if options.md5_chunksize: SCons.Node.FS.File.hash_chunksize = options.md5_chunksize * 1024 @@ -1152,7 +1160,8 @@ def _build_targets(fs, options, targets, target_top): if options.diskcheck: SCons.Node.FS.set_diskcheck(options.diskcheck) - SCons.CacheDir.cache_enabled = not options.cache_disable + SCons.CacheDir.cache_enabled = (not options.cache_disable and + not options.remote_cache_url) SCons.CacheDir.cache_readonly = options.cache_readonly SCons.CacheDir.cache_debug = options.cache_debug SCons.CacheDir.cache_force = options.cache_force @@ -1271,6 +1280,22 @@ def order(dependencies): """Leave the order of dependencies alone.""" return dependencies + if options.remote_cache_fetch_enabled or options.remote_cache_push_enabled: + if not options.remote_cache_url: + raise Exception('--remote-cache-url is required when remote ' + 'caching is enabled.') + + SCons.RemoteCache.raise_if_not_supported() + + remote_cache = SCons.RemoteCache.RemoteCache( + options.remote_cache_connections, + options.remote_cache_url, + options.remote_cache_fetch_enabled, + options.remote_cache_push_enabled, + options.cache_debug) + else: + remote_cache = None + def tmtrace_cleanup(tfile): tfile.close() @@ -1281,7 +1306,8 @@ def tmtrace_cleanup(tfile): atexit.register(tmtrace_cleanup, tmtrace) else: tmtrace = None - taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) + taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace, + remote_cache) # Let the BuildTask objects get at the options to respond to the # various print_* settings, tree_printer list, etc. @@ -1301,7 +1327,8 @@ def tmtrace_cleanup(tfile): # to check if python configured with threads. global num_jobs num_jobs = options.num_jobs - jobs = SCons.Job.Jobs(num_jobs, taskmaster) + jobs = SCons.Job.Jobs(num_jobs, taskmaster, remote_cache, + options.use_scheduler_v2) if num_jobs > 1: msg = None if jobs.num_jobs == 1 or not python_has_threads: diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index def866303f..f2be2706c2 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -142,8 +142,13 @@ def __getattr__(self, attr): 'no_progress', 'num_jobs', 'random', + 'remote_cache_connections', + 'remote_cache_fetch_enabled', + 'remote_cache_push_enabled', + 'remote_cache_url', 'silent', 'stack_size', + 'use_scheduler_v2', 'warn', ] @@ -883,6 +888,27 @@ def opt_implicit_deps(option, opt, value, parser): action="store_true", help="Build dependencies in random order.") + op.add_option('--remote-cache-connections', + dest='remote_cache_connections', default=100, + action='store', nargs=1, type='int', + help='Allow N connections to the server.', + metavar='N') + + op.add_option('--remote-cache-fetch-enabled', + dest='remote_cache_fetch_enabled', default=False, + action='store_true', + help='Whether to fetch nodes from the remote cache') + + op.add_option('--remote-cache-push-enabled', + dest='remote_cache_push_enabled', default=False, + action='store_true', + help='Whether to push nodes to the remote cache') + + op.add_option('--remote-cache-url', + dest='remote_cache_url', default='', + action='store', nargs=1, + help='Server URL for remote caching.') + op.add_option('-s', '--silent', '--quiet', dest="silent", default=False, action="store_true", @@ -950,6 +976,12 @@ def opt_tree(option, opt, value, parser, tree_options=tree_options): help="Search up directory tree for SConstruct, " "build Default() targets from local SConscript.") + op.add_option('--use-scheduler-v2', + dest='use_scheduler_v2', default=False, + action='store_true', + help='Whether to use the more aggressive Parallel scheduler ' + 'on a multi-CPU build.') + def opt_version(option, opt, value, parser): sys.stdout.write(parser.version + '\n') sys.exit(0) diff --git a/SCons/Taskmaster.py b/SCons/Taskmaster.py index d57179545b..bceb78960f 100644 --- a/SCons/Taskmaster.py +++ b/SCons/Taskmaster.py @@ -221,21 +221,8 @@ def execute(self): if not t.retrieve_from_cache(): break cached_targets.append(t) - if len(cached_targets) < len(self.targets): - # Remove targets before building. It's possible that we - # partially retrieved targets from the cache, leaving - # them in read-only mode. That might cause the command - # to fail. - # - for t in cached_targets: - try: - t.fs.unlink(t.get_internal_path()) - except (IOError, OSError): - pass + if not self.process_cached_targets(cached_targets): self.targets[0].build() - else: - for t in cached_targets: - t.cached = 1 except SystemExit: exc_value = sys.exc_info()[1] raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) @@ -249,11 +236,37 @@ def execute(self): buildError.exc_info = sys.exc_info() raise buildError - def executed_without_callbacks(self): + def process_cached_targets(self, cached_targets): + """ + Processes the list of cached targets, updating the task based on + whether all targets were retrieved from cache. + Returns True if all targets were retrieved from cache, False otherwise. + """ + if len(cached_targets) < len(self.targets): + # Remove targets before building. It's possible that we + # partially retrieved targets from the cache, leaving + # them in read-only mode. That might cause the command + # to fail. + # + for t in cached_targets: + try: + t.fs.unlink(t.get_internal_path()) + except (IOError, OSError): + pass + return False + else: + for t in cached_targets: + t.cached = 1 + return True + + def executed_without_callbacks(self, target_infos=None): """ Called when the task has been successfully executed and the Taskmaster instance doesn't want to call the Node's callback methods. + + target_infos is unused. See executed_with_callbacks + documentation for its contents and purpose. """ T = self.tm.trace if T: T.write(self.trace_message('Task.executed_without_callbacks()', @@ -265,7 +278,7 @@ def executed_without_callbacks(self): side_effect.set_state(NODE_NO_STATE) t.set_state(NODE_EXECUTED) - def executed_with_callbacks(self): + def executed_with_callbacks(self, target_infos=None): """ Called when the task has been successfully executed and the Taskmaster instance wants to call the Node's callback @@ -277,12 +290,25 @@ def executed_with_callbacks(self): In any event, we always call "visited()", which will handle any post-visit actions that must take place regardless of whether or not the target was an actual built target or a source Node. + + target_infos is optional. When provided, it is expected to be a list of + the same length as self.targets. Each list entry should be a tuple with + three entries: (1) the target file csig, (2) the target file sha256 + csig, and (3) the target file size. It is expected to be in the same + order as self.targets. """ global print_prepare T = self.tm.trace if T: T.write(self.trace_message('Task.executed_with_callbacks()', self.node)) + if target_infos and len(target_infos) != len(self.targets): + raise Exception('executed_with_callbacks: Unexpected contents of ' + 'target_infos. Expected %d infos, got %d.' % + (len(self.targets), len(target_infos))) + + changed = False + target_infos_index = 0 for t in self.targets: if t.get_state() == NODE_EXECUTING: for side_effect in t.side_effects: @@ -290,7 +316,15 @@ def executed_with_callbacks(self): t.set_state(NODE_EXECUTED) if not t.cached: t.push_to_cache() - t.built() + changed = True + if target_infos: + # This was a remote cache hit, so we already have the node + # size and csig. Avoid unnecessary I/O by providing it now. + (csig, size) = target_infos[target_infos_index] + target_infos_index = target_infos_index + 1 + t.built(csig, size) + else: + t.built() t.visited() if (not print_prepare and (not hasattr(self, 'options') or not self.options.debug_includes)): @@ -298,6 +332,13 @@ def executed_with_callbacks(self): else: t.visited() + # Push the entire task to remote cache now that all targets have been + # marked as built. That clears memoizations, which allows us to + # properly retrieve csigs. + if (changed and self.tm.remote_cache and + self.tm.remote_cache.push_enabled): + self.tm.remote_cache.push_task(self) + executed = executed_with_callbacks def failed(self): @@ -546,6 +587,12 @@ def _exception_raise(self): # raise e.__class__, e.__class__(e), sys.exc_info()[2] # exec("raise exc_type(exc_value).with_traceback(exc_traceback)") + def is_cacheable(self): + """Checks whether all targets for the task are cacheable.""" + for t in self.targets: + if not t.should_retrieve_from_cache(): + return False + return True class AlwaysTask(Task): @@ -592,7 +639,8 @@ class Taskmaster: The Taskmaster for walking the dependency DAG. """ - def __init__(self, targets=[], tasker=None, order=None, trace=None): + def __init__(self, targets=[], tasker=None, order=None, trace=None, + remote_cache=None): self.original_top = targets self.top_targets_left = targets[:] self.top_targets_left.reverse() @@ -607,6 +655,7 @@ def __init__(self, targets=[], tasker=None, order=None, trace=None): self.trace = trace self.next_candidate = self.find_next_candidate self.pending_children = set() + self.remote_cache = remote_cache def find_next_candidate(self): """ @@ -1027,6 +1076,10 @@ def cleanup(self): """ Check for dependency cycles. """ + if self.remote_cache: + self.remote_cache.close() + self.remote_cache = None + if not self.pending_children: return diff --git a/bin/files b/bin/files index 08b1caa3bd..bece30fff7 100644 --- a/bin/files +++ b/bin/files @@ -26,6 +26,7 @@ ./SCons/Platform/posix.py ./SCons/Platform/sunos.py ./SCons/Platform/win32.py +./SCons/RemoteCache.py ./SCons/Scanner/C.py ./SCons/Scanner/D.py ./SCons/Scanner/Fortran.py diff --git a/setup.cfg b/setup.cfg index c39f2de28e..c352bb7cda 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ zip_safe = False python_requires = >=3.5 install_requires = setuptools + urllib3 >= 1.8 setup_requires = setuptools include_package_data = True diff --git a/test/RemoteCache/.exclude_tests b/test/RemoteCache/.exclude_tests new file mode 100644 index 0000000000..1fc8fab382 --- /dev/null +++ b/test/RemoteCache/.exclude_tests @@ -0,0 +1,2 @@ +RemoteCacheTestServer.py +RemoteCacheUtils.py diff --git a/test/RemoteCache/CachePushAndFetch.py b/test/RemoteCache/CachePushAndFetch.py new file mode 100644 index 0000000000..102b9a3cba --- /dev/null +++ b/test/RemoteCache/CachePushAndFetch.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Tests remote cache push and then fetch. In this test, cache misses are +expected for the first compilation with cache hits for the second compilation. +This is because the first compilation is the cache producer for the second +compilation. +""" + +import os +import stat +import sys + +import RemoteCacheUtils +import TestSCons + +test = TestSCons.TestSCons() +RemoteCacheUtils.skip_test_if_no_urllib3(test) +test.file_fixture('test_main.c') +test.dir_fixture('CachePushAndFetch') +server_url = RemoteCacheUtils.start_test_server(test.workpath()) + +arguments = [ + '--remote-cache-fetch-enabled', + '--remote-cache-push-enabled', + '--remote-cache-url=' + server_url, + '--cache-debug=%s' % test.workpath('cache.txt'), +] + +# Populate the cache. The expected compiler output depends on the platform. +# TODO: Do we need to call SCons.Tool.MSCommon.msvc_exists() on Windows and +# handle any other compilers? +if sys.platform == 'win32': + expected_compiler_output = """\ +cl /Fotest_main.obj /c test_main.c /nologo +test_main.c +link /nologo /OUT:main.exe test_main.obj""" +else: + expected_compiler_output = """\ +gcc -o test_main.o -c test_main.c +gcc -o main test_main.o""" + +test.run(arguments=arguments, + stdout=test.wrap_stdout("""\ +{expected_compiler_output} +RemoteCache: 0.0 percent cache hit rate on 2 cacheable tasks with 0 hits, 2 \ +misses, 0 w/cache suspended. 66.7 percent of total tasks cacheable, due to \ +1/3 tasks marked not cacheable. Saw 0 total failures, 0 cache restarts. +""".format(expected_compiler_output=expected_compiler_output))) + +# Clean the build directory. +test.run(arguments='-c .') + +# Run and confirm that we had cache hits. +test.run(arguments=arguments, + stdout=test.wrap_stdout("""\ +RemoteCache: 100.0 percent cache hit rate on 2 cacheable tasks with 2 hits, \ +0 misses, 0 w/cache suspended. 66.7 percent of total tasks cacheable, due to \ +1/3 tasks marked not cacheable. Saw 0 total failures, 0 cache restarts. +""")) + +# Confirm that cache hits set the execute bits only for the executable. +if os.name == 'posix': + object_mode = os.stat(test.workpath('test_main.o')).st_mode + executable_mode = os.stat(test.workpath('main')).st_mode + executable_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + assert object_mode & executable_bits == 0, object_mode + assert executable_mode & executable_bits == executable_bits, \ + executable_mode + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/RemoteCache/CachePushAndFetch/SConstruct b/test/RemoteCache/CachePushAndFetch/SConstruct new file mode 100644 index 0000000000..d5b063de33 --- /dev/null +++ b/test/RemoteCache/CachePushAndFetch/SConstruct @@ -0,0 +1,2 @@ +env = Environment() +env.Program('main', 'test_main.c') \ No newline at end of file diff --git a/test/RemoteCache/RemoteCacheTestServer.py b/test/RemoteCache/RemoteCacheTestServer.py new file mode 100644 index 0000000000..5522a6e55c --- /dev/null +++ b/test/RemoteCache/RemoteCacheTestServer.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +RemoteCacheTestServer.py + +Test Python script that acts like a Bazel remote cache server. +""" + +import argparse +import hashlib +import http.server +import os + +parser = argparse.ArgumentParser( + description='Test loopback remote cache server') +parser.add_argument('address', help='Address to listen on') +parser.add_argument('port', type=int, help='Port to listen on') +args = vars(parser.parse_args()) + + +class HandlerClass(http.server.SimpleHTTPRequestHandler): + """ + Subclass of SimpleHTTPRequestHandler to handle PUT and GET requests, which + are the only requests that the SCons remote caching code makes. + SimpleHTTPRequestHandler automatically supports GET requests, so we only + need to implement PUT requests. http.client has code to handle a request + XYZ by calling the do_XYZ function, so we only need to implement do_PUT. + """ + def do_PUT(self): + path = self.translate_path(self.path) + dir, file = os.path.split(path) + if not os.path.exists(dir): + os.makedirs(dir) + + length = int(self.headers['Content-Length']) + data = self.rfile.read(length) + + if os.path.basename(dir) == 'cas': + # For Content Addressable Storage entries, validate that the last + # path segment matches the sha256 of the content; return + # Unprocessable Entity otherwise. + actual_sha256 = hashlib.sha256(data).hexdigest() + if file != actual_sha256: + self.send_response(422) + self.end_headers() + return + + with open(path, 'wb') as f: + f.write(data) + + # No Content is a standard answer for a successful resource PUT. + self.send_response(204) + self.end_headers() + +with http.server.HTTPServer((args['address'], args['port']), + HandlerClass) as httpd: + httpd.serve_forever() diff --git a/test/RemoteCache/RemoteCacheUtils.py b/test/RemoteCache/RemoteCacheUtils.py new file mode 100644 index 0000000000..d941da1979 --- /dev/null +++ b/test/RemoteCache/RemoteCacheUtils.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Test utilities shared between remote cache tests. +""" + +import atexit +import os +import random +import subprocess +import sys + + +def start_test_server(directory): + """ + Starts a test script which pretends to be a Bazel remote cache server + Returns the full url to the test server, which should be passed into SCons + using --remote-cache-url + """ + host = 'localhost' + port = str(random.randint(49152, 65535)) + process = subprocess.Popen( + [sys.executable, + os.path.join(os.path.dirname(__file__), 'RemoteCacheTestServer.py'), + host, port], + cwd=directory, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def ShutdownServer(): + process.kill() + atexit.register(ShutdownServer) + + return 'http://%s:%s' % (host, port) + + +def skip_test_if_no_urllib3(test): + """Skips the test if urllib3 is not available""" + try: + import urllib3 # noqa: F401 + except ImportError: + test.skip_test('urllib3 not found; skipping test') From 79831d2b5de0e611b241e0d8a07a2c610499236f Mon Sep 17 00:00:00 2001 From: Adam Gross Date: Tue, 13 Jul 2021 08:52:55 -0400 Subject: [PATCH 2/3] Fix some sider issues Some are inherited simply from moving code around, but this fixes the issues my change introduced. --- SCons/Job.py | 1 - SCons/RemoteCache.py | 3 --- SCons/RemoteCacheTests.py | 10 +++------- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/SCons/Job.py b/SCons/Job.py index b8d76b056d..bc94b6d20c 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -30,7 +30,6 @@ import SCons.compat import SCons.Node -from collections import deque import os import signal diff --git a/SCons/RemoteCache.py b/SCons/RemoteCache.py index faee8e109b..ec9bac8b72 100644 --- a/SCons/RemoteCache.py +++ b/SCons/RemoteCache.py @@ -71,7 +71,6 @@ from concurrent.futures import ThreadPoolExecutor -import atexit import datetime import json import os @@ -580,8 +579,6 @@ def _fetch(self, task): This function can be called multiple times from different threads in the ThreadPoolExecutor, so it must be thread-safe. """ - fetched_all_nodes = False - try: url = self._get_task_cache_url(task) diff --git a/SCons/RemoteCacheTests.py b/SCons/RemoteCacheTests.py index 4c3505fc36..964ea18b0f 100644 --- a/SCons/RemoteCacheTests.py +++ b/SCons/RemoteCacheTests.py @@ -27,14 +27,10 @@ import os import queue import stat -import sys import unittest -from time import sleep from unittest import mock -from TestCmd import TestCmd import TestSCons -import TestUnit import SCons.RemoteCache @@ -138,7 +134,7 @@ def CreateRemoteCache(mock_urllib3, worker_count, server_address, cache = SCons.RemoteCache.RemoteCache( worker_count, server_address, fetch_enabled, push_enabled, None) mock_urllib3.util.parse_url.assert_called_with(server_address) - assert cache != None, cache + assert cache is not None, cache q = queue.Queue(0) cache.set_fetch_response_queue(q) @@ -317,7 +313,7 @@ def runTest(self, mock_urllib3): assert task == task2 assert not t.cached assert not hit - assert target_infos == None + assert target_infos is None assert not cache.connection_pool.pending_responses, \ cache.connection_pool.pending_responses @@ -343,7 +339,7 @@ def runTest(self, mock_urllib3): assert task == task2 assert not t.cached assert not hit - assert target_infos == None + assert target_infos is None assert not cache.connection_pool.pending_responses, \ cache.connection_pool.pending_responses From a836a859cea235467a5aa36f8874988ebfb831cf Mon Sep 17 00:00:00 2001 From: Adam Gross Date: Tue, 17 Aug 2021 10:18:32 -0400 Subject: [PATCH 3/3] Fix Ninja tests --- SCons/Tool/ninja/Overrides.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index eb8dbb5cb2..23c740a968 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -73,7 +73,8 @@ def _print_cmd_str(*_args, **_kwargs): pass -def ninja_always_serial(self, num, taskmaster): +def ninja_always_serial(self, num, taskmaster, remote_cache=None, + use_scheduler_v2=False): """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" # We still set self.num_jobs to num even though it's a lie. The # only consumer of this attribute is the Parallel Job class AND