diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml new file mode 100644 index 000000000..1b27aa454 --- /dev/null +++ b/.github/workflows/license-header-check.yml @@ -0,0 +1,50 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A workflow to check copyright/license header +name: license header check + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + license-header-check: + runs-on: ubuntu-latest + if: "!contains(github.event.pull_request.title, '[bot]')" + steps: + - name: Get checkout depth + run: | + echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 10 ))" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: ${{ env.PR_FETCH_DEPTH }} + + - name: license-header-check + uses: NVIDIA/spark-rapids-common/license-header-check@main + with: + included_file_patterns: | + *.py, + *.toml, + *.ini, + *.yml, + *.yaml, + *.sh, + *.properties, + *.xml, + *.feature, + *.scala + \ No newline at end of file diff --git a/core/src/test/resources/worker_info-autotuner-example-8cores-16gmemory.yaml b/core/src/test/resources/worker_info-autotuner-example-8cores-16gmemory.yaml index 9ad530544..12091e726 100644 --- a/core/src/test/resources/worker_info-autotuner-example-8cores-16gmemory.yaml +++ b/core/src/test/resources/worker_info-autotuner-example-8cores-16gmemory.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + system: numCores: 8 memory: 16384MiB diff --git a/data_validation/pyproject.toml b/data_validation/pyproject.toml index 0516a2b98..8bff2bb93 100644 --- a/data_validation/pyproject.toml +++ b/data_validation/pyproject.toml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/data_validation/templates/datavalid_conf.yml b/data_validation/templates/datavalid_conf.yml index 5fe2d1f31..8c864d1fe 100644 --- a/data_validation/templates/datavalid_conf.yml +++ b/data_validation/templates/datavalid_conf.yml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + sparkConf: spark.executor.memory: 4g spark.executor.cores: 3 diff --git a/data_validation/tox.ini b/data_validation/tox.ini index 58c3a0212..4725be6de 100644 --- a/data_validation/tox.ini +++ b/data_validation/tox.ini @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" diff --git a/user_tools/pyproject.toml b/user_tools/pyproject.toml index c92bff303..2c903919f 100644 --- a/user_tools/pyproject.toml +++ b/user_tools/pyproject.toml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/user_tools/src/spark_rapids_pytools/resources/bootstrap-conf.yaml b/user_tools/src/spark_rapids_pytools/resources/bootstrap-conf.yaml index ee24afdd4..a339a4d0f 100644 --- a/user_tools/src/spark_rapids_pytools/resources/bootstrap-conf.yaml +++ b/user_tools/src/spark_rapids_pytools/resources/bootstrap-conf.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: shortName: 'boot' # workDir: rapids_user_tools_bootstrap diff --git a/user_tools/src/spark_rapids_pytools/resources/cluster-configs.yaml b/user_tools/src/spark_rapids_pytools/resources/cluster-configs.yaml index 700c181a5..d64b1ff95 100644 --- a/user_tools/src/spark_rapids_pytools/resources/cluster-configs.yaml +++ b/user_tools/src/spark_rapids_pytools/resources/cluster-configs.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + clusterConfigs: constants: # Maximum amount of pinned memory to use per executor in megabytes diff --git a/user_tools/src/spark_rapids_pytools/resources/dev/process_databricks_azure_pricing.py b/user_tools/src/spark_rapids_pytools/resources/dev/process_databricks_azure_pricing.py index 19953f393..9ca2ddc1c 100644 --- a/user_tools/src/spark_rapids_pytools/resources/dev/process_databricks_azure_pricing.py +++ b/user_tools/src/spark_rapids_pytools/resources/dev/process_databricks_azure_pricing.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Python script used to process Databricks Azure instance pricing information.""" import json diff --git a/user_tools/src/spark_rapids_pytools/resources/diagnostic-conf.yaml b/user_tools/src/spark_rapids_pytools/resources/diagnostic-conf.yaml index 359412120..7786e2aae 100644 --- a/user_tools/src/spark_rapids_pytools/resources/diagnostic-conf.yaml +++ b/user_tools/src/spark_rapids_pytools/resources/diagnostic-conf.yaml @@ -1,2 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: shortName: 'diag' diff --git a/user_tools/src/spark_rapids_pytools/resources/profiling-conf.yaml b/user_tools/src/spark_rapids_pytools/resources/profiling-conf.yaml index 847e33914..b183a332f 100644 --- a/user_tools/src/spark_rapids_pytools/resources/profiling-conf.yaml +++ b/user_tools/src/spark_rapids_pytools/resources/profiling-conf.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + toolOutput: subFolder: rapids_4_spark_profile textFormat: diff --git a/user_tools/src/spark_rapids_pytools/resources/qualification-conf.yaml b/user_tools/src/spark_rapids_pytools/resources/qualification-conf.yaml index 8e998011c..c149669c5 100644 --- a/user_tools/src/spark_rapids_pytools/resources/qualification-conf.yaml +++ b/user_tools/src/spark_rapids_pytools/resources/qualification-conf.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + toolOutput: completeOutput: true subFolder: rapids_4_spark_qualification_output diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc/cpu-00.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc/cpu-00.yaml index c6bc6038d..9a0d74652 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc/cpu-00.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc/cpu-00.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + clusterName: dataproc-test-nongpu-cluster clusterUuid: 11111111-1111-1111-1111-111111111111 config: diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc_gke/cpu-00.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc_gke/cpu-00.yaml index 39bd70ddf..405f6bea5 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc_gke/cpu-00.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/dataproc_gke/cpu-00.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + clusterName: dataproc-gke-test-nongpu-cluster clusterUuid: 11111111-1111-1111-1111-111111111111 config: diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/onprem/cpu-00.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/onprem/cpu-00.yaml index b170d85b0..9a900ae95 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/cluster/onprem/cpu-00.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/cluster/onprem/cpu-00.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + config: masterConfig: numCores: 2 diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_00.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_00.yaml index aa566a71f..b656ea7ee 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_00.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_00.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This yaml file is a configuration file for a tool that uses Spark 3.5.0 as dependency # invalid: API version is smaller than accepted API version api_version: '0.9' diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_01.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_01.yaml index a723f2e66..dd713e594 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_01.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_01.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This yaml file is a configuration file # invalid: empty file # Error:1 validation error for ToolsConfig diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_02.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_02.yaml index 210b4419a..dc8f1df49 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_02.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/invalid/tools_config_inv_02.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This yaml file is a configuration file for a tool that uses Spark 3.5.0 as dependency # invalid: Local file can be verified during the initialization and it should fail because no such # file exists. diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/valid/tools_config_00.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/valid/tools_config_00.yaml index f7dccf951..85ef4e15b 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/valid/tools_config_00.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/tools_config/valid/tools_config_00.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This yaml file is a configuration file for a tool that uses Spark 3.5.0 as dependency # Minimal file content. api_version: '1.0' diff --git a/user_tools/tests/spark_rapids_tools_ut/resources/worker_info.yaml b/user_tools/tests/spark_rapids_tools_ut/resources/worker_info.yaml index d9aaa14d5..7b3f8f32b 100644 --- a/user_tools/tests/spark_rapids_tools_ut/resources/worker_info.yaml +++ b/user_tools/tests/spark_rapids_tools_ut/resources/worker_info.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + system: numCores: 32 memory: 212992MiB diff --git a/user_tools/tox.ini b/user_tools/tox.ini index b7a766d98..f89d18857 100644 --- a/user_tools/tox.ini +++ b/user_tools/tox.ini @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox"