[ci] feat: add more CI workflow

.github/workflows/gpu_test.yml → .github/workflows/dataset.yml

@@ -1,4 +1,4 @@
-name: ray
+name: dataset
 
 on:
   # Trigger the workflow on push or pull request,
@@ -8,13 +8,13 @@ on:
       - main
     paths:
       - "**/*.py"
-      - .github/workflows/ray_test.yml
+      - .github/workflows/dataset.yml
   pull_request:
     branches:
       - main
     paths:
       - "**/*.py"
-      - .github/workflows/ray_test.yml
+      - .github/workflows/dataset.yml
 
 jobs:
   ray:
@@ -30,7 +30,7 @@ jobs:
         run: |
           [ ! -d "$HOME/verl-data" ] && git clone --depth 1 https://github.com/eric-haibin-lin/verl-data ~/verl-data
           pytest -s -x tests/verl
-      - name: Running ray tests that need 2 GPUs
+      - name: Running ray test using cupy (move it to L20 when dockerfile ready)
         run: |
           cd tests/ray
-          pytest -s -x test_rvdz.py test_driverfunc_to_worker.py test_data_transfer.py test_colocated_workers.py test_check_worker_alive.py
+          pytest -s -x test_rvdz.py

.github/workflows/e2e_gpu.yml

@@ -0,0 +1,38 @@
+name: e2e_gpu
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/e2e_gpu.yml
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/e2e_gpu.yml
+
+jobs:
+  e2e_gpu:
+    runs-on: [self-hosted, l20-1]
+    env:
+      HTTP_PROXY: ${{ secrets.PROXY_HTTP }}
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1"
+    container:
+      image: verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3
+      options: --gpus all --shm-size=10g
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+            fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install -e .[test]
+      - name: Running digit completon e2e training tests on 8 L20 GPUs
+        run: |
+          bash tests/e2e/run_ray_trainer.sh

.github/workflows/ray_test.yml

@@ -0,0 +1,42 @@
+name: ray
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/ray_test.yml
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/ray_test.yml
+
+jobs:
+  ray:
+    runs-on: [self-hosted, l20-0]
+    env:
+      HTTP_PROXY: ${{ secrets.PROXY_HTTP }}
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1"
+      HF_HUB_ENABLE_HF_TRANSFER: 1
+    container:
+      image: verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3
+      options: --gpus all --shm-size=10g
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+            fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip install hf_transfer
+          pip install -e .[test]
+          pip install --upgrade "ray>=2.40.0"
+      - name: Running ray tests that need 8 GPUs
+        run: |
+          cd tests/ray
+          pytest -s -x --ignore=test_check_worker_alive.py --ignore=test_rvdz.py .

.github/workflows/sanity.yml

@@ -0,0 +1,39 @@
+name: sanity
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/sanity.yml
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/sanity.yml
+
+jobs:
+  sanity:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install the current repository
+        run: |
+          pip install -e .[test]
+      - name: Run sanity test
+        run: |
+          pytest -s -x tests/sanity
+      - name: Run untility test
+        run: |
+          pytest -s -x tests/utility

.github/workflows/vllm.yml

@@ -0,0 +1,42 @@
+name: vllm
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/vllm.yml
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**/*.py"
+      - .github/workflows/vllm.yml
+
+jobs:
+  vllm:
+    runs-on: [self-hosted, l20-0]
+    env:
+      HTTP_PROXY: ${{ secrets.PROXY_HTTP }}
+      HTTPS_PROXY: ${{ secrets.PROXY_HTTPS }}
+      NO_PROXY: "localhost,127.0.0.1"
+      HF_HUB_ENABLE_HF_TRANSFER: 1
+    container:
+      image: verlai/verl:vemlp-th2.4.0-cu124-vllm0.6.3-ray2.10-te1.7-v0.0.3
+      options: --gpus all --shm-size=10g
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+            fetch-depth: 0
+      - name: Install the current repository
+        run: |
+          pip3 install hf_transfer
+          pip3 install -e .[test]
+          pip3 install vllm==0.5.4
+      - name: Running vllm tests on 8 L20 GPUs
+        run: |
+          cd tests/rollout
+          torchrun --standalone --nnodes=1 --nproc_per_node=8 $(which pytest) -s test_vllm_hf_loader.py

.github/workflows/yapf_format.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install yapf
+          pip install --upgrade yapf
           pip install toml==0.10.2
       - name: Running yapf
         run: |

requirements.txt

@@ -4,6 +4,7 @@ datasets
 dill
 hydra-core
 numpy
+pandas
 pybind11
 ray
 tensordict<0.6

tests/e2e/arithmetic_sequence/data/create_dataset.py

@@ -0,0 +1,46 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tests.e2e.envs.digit_completion import DigitCompletion, generate_ground_truth_response
+from torch.utils import data
+import os
+
+if __name__ == '__main__':
+    simple_task = DigitCompletion(max_number=9, max_diff=9, max_num_in_response=9)
+    all_prompts = simple_task.get_all_prompts()
+
+    # 21 * 6 * 4
+    train_data, test_data = data.random_split(all_prompts, lengths=[0.8, 0.2])
+    train_data = list(train_data)
+    test_data = list(test_data)
+
+    train_data = [[{'role': 'user', 'content': str(item)}] \
+                     for item in train_data]
+    test_data = [[{'role': 'user', 'content': str(item)}] \
+                     for item in test_data]
+
+    print(f'Size of train: {len(train_data)}, size of test: {len(test_data)}')
+
+    train_data = {'prompt': train_data}
+    test_data = {'prompt': test_data}
+
+    model_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+
+    import pandas as pd
+
+    train_data_frame = pd.DataFrame(train_data)
+    test_data_frame = pd.DataFrame(test_data)
+
+    train_data_frame.to_parquet(os.path.join(model_folder, 'train.parquet'))
+    test_data_frame.to_parquet(os.path.join(model_folder, 'test.parquet'))

tests/e2e/arithmetic_sequence/model/config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "eos_token_id": 1,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 344,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 4,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": true,
+  "vocab_size": 16
+}

tests/e2e/arithmetic_sequence/model/create_model_tokenizer.py

@@ -0,0 +1,61 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Create a random model and tokenizer for PPO training
+"""
+
+import torch
+import os
+from transformers import AutoModelForCausalLM, LlamaConfig, AutoTokenizer
+
+from tests.e2e.envs.digit_completion import CharTokenizer
+
+tokenizer = CharTokenizer(
+    characters=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ',', ':'],
+    model_max_length=2048,
+    chat_template=
+    "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ sep_token }}{% endif %}"
+)
+
+config = LlamaConfig(vocab_size=(tokenizer.vocab_size + 16 - 1) // 16 * 16,
+                     hidden_size=128,
+                     intermediate_size=344,
+                     num_hidden_layers=4,
+                     num_attention_heads=4,
+                     num_key_value_heads=4,
+                     pad_token_id=tokenizer.pad_token_id,
+                     bos_token_id=tokenizer.bos_token_id,
+                     eos_token_id=tokenizer.eos_token_id)
+
+model = AutoModelForCausalLM.from_config(config, torch_dtype=torch.bfloat16)
+
+model_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+os.makedirs(model_folder, exist_ok=True)
+
+model.save_pretrained(model_folder)
+
+tokenizer_folder = model_folder
+tokenizer.save_pretrained(tokenizer_folder)
+
+load_tokenizer = AutoTokenizer.from_pretrained(tokenizer_folder)
+
+chat = [{'role': 'user', 'content': '1,0:2,3'}]
+
+load_tokenizer.padding_side = 'left'
+print(
+    load_tokenizer.apply_chat_template(chat,
+                                       tokenize=True,
+                                       add_generation_prompt=True,
+                                       max_length=10,
+                                       padding='max_length'))

tests/e2e/arithmetic_sequence/model/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 1,
+  "pad_token_id": 2,
+  "transformers_version": "4.43.3"
+}

tests/e2e/arithmetic_sequence/model/tokenizer_config.json

@@ -0,0 +1,18 @@
+{
+    "char_ords": [
+        48,
+        49,
+        50,
+        51,
+        52,
+        53,
+        54,
+        55,
+        56,
+        57,
+        44,
+        58
+    ],
+    "model_max_length": 2048,
+    "chat_template": "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set role = message['role'] %}{{ message['content'] }}{% endfor %}{% if add_generation_prompt %}{{ sep_token }}{% endif %}"
+}