Llm eloquence2

.github/workflows/test_llm.yml

@@ -0,0 +1,42 @@
+name: UnitTests for Fine-tuning LLMs
+
+on:
+  pull_request:
+    types: [opened, synchronize, edited]
+
+jobs:
+  run:
+    if: false == contains(github.event.pull_request.title, 'WIP')
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9']
+        torch-version: ['2.0.0']
+        torchvision-version: ['0.15.0']
+        torchaudio-version: ['2.0.0']
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: '3.9'
+    steps:
+    - uses: actions/checkout@master
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install PyTorch ${{ matrix.torch-version }}+cpu
+      run: |
+        pip install numpy typing-extensions dataclasses
+        pip install torch==${{ matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install FS
+      run: |
+        pip install -e .[llm,test]
+    - name: Test GPT2
+      run: |
+        python federatedscope/main.py --cfg federatedscope/llm/baseline/testcase.yaml federate.total_round_num 1 eval.count_flops False train.local_update_steps 2 data.splits "[0.998, 0.001, 0.001]"
+        [ $? -eq 1 ] && exit 1 || echo "Passed"
+    - name: Test GPT2 with offsite-tuning
+      run: |
+        python federatedscope/main.py --cfg federatedscope/llm/baseline/testcase.yaml federate.total_round_num 1 eval.count_flops False llm.offsite_tuning.use True llm.offsite_tuning.emu_l 2 llm.offsite_tuning.emu_r 10 train.local_update_steps 2 data.splits "[0.998, 0.001, 0.001]"
+        [ $? -eq 1 ] && exit 1 || echo "Passed"

LICENSE

@@ -661,3 +661,45 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+---------------------------------------------------------------------------------
+The implementations of LLM dataset in federatedscope/llm/dataset/llm_dataset.py
+adapted from https://github.com/tatsu-lab/stanford_alpaca (Apache License)
+
+Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+---------------------------------------------------------------------------------
+The implementations of evaluation for MMLU in federatedscope/llm/eval_for_mmlu/eval.py
+and federatedscope/llm/eval_for_mmlu/categories.py are adapted from https://github.com/hendrycks/test (MIT License)
+
+Copyright (c) 2020 Dan Hendrycks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1 @@
+federatedscope/llm/README.md

README_setup.md

@@ -0,0 +1,89 @@
+# Installation, Setup and Running of FederatedScope for LLMs Fine-tuning
+
+First, use a virtual environment manager such as pyenv to create a virtual environment. Make sure you are using Python 3.9.0:
+
+```bash
+pyenv install 3.9.0
+pyenv virtualenv 3.9.0 fs-llm_3.9.0
+pyenv activate fs-llm_3.9.0
+```
+
+Clone the specific branch of the FederatedScope repository to your machine:
+
+```bash
+git clone --branch llm-eloquence https://github.com/jordiluque/FederatedScope.git
+```
+
+To ensure that the correct CUDA paths are set, add the following lines to your `.bashrc` (or equivalent shell configuration file). The CUDA version should be around version 12 (e.g., 12.4, 12.5, or 12.6). If you don’t already have the [CUTLASS](https://github.com/NVIDIA/cutlass) repository installed, clone and set it up on your machine.
+
+```bash
+export PATH=/usr/local/cuda-12/bin/:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-12/lib64:/usr/local/cuda-12/lib:$LD_LIBRARY_PATH
+export CUDA_HOME=/usr/local/cuda-12
+export CUTLASS_PATH=/home/user/repos/cutlass 
+```
+
+After editing `.bashrc`, don't forget to run:
+
+```bash
+source ~/.bashrc
+```
+
+Install the following Python libraries. The specific versions are known to work well:
+```bash
+pip install torch==2.4.1 torchaudio==2.4.1 torchvision==0.19.1
+```
+
+From the source of the repository, install the required FederatedScope requirements:
+```bash
+pip install -e .[llm]
+```
+
+Check if the default script runs correctly to verify the installation:
+
+```bash
+python federatedscope/main.py --cfg federatedscope/llm/baseline/testcase.yaml
+```
+
+Now let's install and configure DeepSpeed. It is is highly recommended for efficiently fine-tuning LLMs. To install it, run:
+
+```bash
+pip install deepspeed
+```
+
+Install the cupy library for CUDA Acceleration with CUDA 12 support:
+
+```bash
+pip install cupy-cuda12x
+```
+
+If you are working with recent models (e.g., Phi models), they may not be included in the default version of the transformers library. In this case, upgrade the library:
+
+```bash
+pip install --upgrade transformers
+```
+
+Before using DeepSpeed, review the configuration file at `federatedscope/llm/baseline/deepspeed/ds_config_4bs.json`. Ensure that the train_batch_size parameter is properly set to match the number of GPUs available on your machine.
+
+Check if fine-tuning an LLM in standalone mode works correctly with DeepSpeed. Run the following script to verify that the fine-tuning process is functioning properly:
+
+```bash
+deepspeed federatedscope/main.py --cfg configs/standalone/Phi-3.5-mini-instruct/ds_3c_200r_30ls.yaml
+```
+
+To execute federated fine-tuning in distributed mode, separate commands need to be run for the server and each client. In the FederatedScope framework, each client must run on a different machine. The following config files will allow us to test if the setup works with two clients in distributed mode. However, before running the commands, ensure that the `server_host`, `server_port`, `client_host`, and `client_port` fields in the config files are updated with the correct IP addresses and ports for your machines. Additionally, adjust CUDA_VISIBLE_DEVICES to reflect the number of GPUs available on each machine.
+
+To run the server use:
+```bash
+deepspeed --master_addr=127.0.0.1 --master_port=29500 federatedscope/main.py --cfg configs/distributed/Phi-3.5-mini-instruct/server_ds_2c_200r_30ls.yaml
+```
+
+To run a first client in one machine use:
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2 deepspeed --master_addr=127.0.0.1 --master_port=29500 federatedscope/main.py --cfg configs/distributed/Phi-3.5-mini-instruct/client_1_ds_2c_200r_30ls.yaml
+```
+
+To run a second client in another machine:
+```bash
+CUDA_VISIBLE_DEVICES=0,1,2 deepspeed --master_addr=127.0.0.1 --master_port=29500 federatedscope/main.py --cfg configs/distributed/Phi-3.5-mini-instruct/client_2_ds_2c_200r_30ls.yaml 
+```

benchmark/Backdoor-bench/README.md

@@ -0,0 +1,91 @@
+# Benchmark for Back-door Attack on Personalized Federated Learning
+
+
+
+Backdoor-bench is a benchmark for backdoor attacks on personalized federated learning. It contains backdoor attacks including [edge-based trigger](https://arxiv.org/abs/2007.05084), [BadNet](https://ieeexplore.ieee.org/document/8685687), [Blended](https://arxiv.org/abs/1712.05526) and [SIG](https://arxiv.org/abs/1902.11237). The attacked pFL methods include: FedAvg, Fine-tuning (FT), Ditto, FedEM, pFedMe, FedBN, FedRep. More details about the benchmark settings and experimental results refer to our KDD [paper](https://arxiv.org/abs/2302.01677). 
+
+**Notice**:
+Considering FederatedScope is an open-sourced library that updates frequently, to ensure the reproducibility of the experimental results, we create a new branch `backdoor-bench`. The users can reproduce the results by running the configs under the directory [scripts/B-backdoor_scripts attack_config](https://github.com/alibaba/FederatedScope/tree/backdoor-bench/scripts/backdoor_scripts/attack_config). The results of our paper is located in `paper_plot/results_all`.
+
+## Publications
+
+If you find Back-door-bench useful for your research or development, please cite the following [paper](https://arxiv.org/pdf/2302.01677.pdf):
+
+```tex
+@inproceedings{
+qin2023revisiting,
+title={Revisiting Personalized Federated Learning: Robustness Against Backdoor Attacks},
+author={Zeyu Qin and Liuyi Yao and Daoyuan Chen and Yaliang Li and Bolin Ding and Minhao Cheng},
+booktitle={29th SIGKDD Conference on Knowledge Discovery and Data Mining - Applied Data Science Track},
+year={2023},
+}
+```
+
+## Quick Start
+
+To run the script, you should 
+- First clone the repository [FederatedScope](https://github.com/alibaba/FederatedScope),
+- Then follow [README.md](https://github.com/alibaba/FederatedScope/blob/master/README.md) to build the running environment for FederatedScope, 
+- Switch to the branch `backdoor-bench` and run the scripts
+```bash
+# Step-1. clone the repository 
+git clone https://github.com/alibaba/FederatedScope.git
+
+# Step-2. follow https://github.com/alibaba/FederatedScope/blob/master/README.md to build the running environment
+
+# Step-3. install packages required by the benchmark
+pip install opencv-python matplotlib pympler scikit-learn
+
+# Step-3. switch to the branch `backdoor-bench` for the benchmark
+git fetch
+git switch backdoor-bench
+
+# Step-4. run the baseline (taking attacking FedAvg with Edge type trigger as an example)
+cd FederatedScope
+python federatedscope/main.py --cfg scripts/backdoor_scripts/attack_config/backdoor_fedavg_resnet18_on_cifar10_small.yaml
+
+```
+## Reimplementing Results of Paper
+
+The all scripts of conducting experiments are in file [attack_config](https://github.com/alibaba/FederatedScope/tree/backdoor-bench/scripts/backdoor_scripts/attack_config). 
+- **Backdoor or not**: Files with 'backdoor' in their filename are experimental instructions related to backdoor poisoning during the training process. Files without 'backdoor' are experimental instructions about normal FL or pFL training process.  
+- **Models**: Files with different models name represents experiments with using different models, such as "convnet" or "resnet18".
+- **Datasets**: Files with different dataset name represents experiments on different datasets, such as "femnist" or "cifar10".
+- **pFL Methods**: Files with different method name represents experiments with using different pFL methods. 
+- **IID vs Non-IID**: Files with 'iid' represents experiments under IID settings. 
+- **Ablation Study**: Files with 'abl' represents ablation studies of pFL methods conducted in Section 5. 
+- **FedBN**: Files with 'bn' and 'para' or 'sta' mean experiments of Fed-para and Fed-sta conducted in Section 5.1. 
+- **Existing Defense**: Experiments about existing defense methods:
+    *  Krum: please set attack.krum: True
+    *  Multi-Krum: please set attack.multi_krum: True
+    *  Norm_clip: please set attack.norm_clip: True and tune attack.norm_clip_value. 
+    *  Adding noise: please tune attack.dp_noise. 
+
+**Notice:** The Files with 'small' or 'avg' are about experiments with changing attackers since we wish to test whether the size of the local dataset possessed by the attacker will have an impact on the success of the backdoor poisoning. You can ignore them. 
+
+----
+
+## Explanations about Attack Config
+
+
+    attack:
+        setting: 'fix' --fix-frequency attack setting
+        freq: 10 --the adversarial client is selected for every fixed 10 round.
+        attack_method: 'backdoor'
+        attacker_id: 15 --the client id of attacker
+        label_type: 'dirty' --dirty or clean-label attacks. We now only support dirty-label attacks
+        trigger_type: gridTrigger --BadNet: gridTrigger; Blended: hkTrigger; edge: edge; SIG: sigTrigger
+        edge_num: 500 --the number of samples with edge trigger
+        poison_ratio: 0.5 --poisoning ratio of local training dataset
+        target_label_ind: 9 --target label of backdoor attacks
+        self_opt: False --you can ignore it since we do not test it. 
+        self_lr: 0.1 --you can ignore it since we do not test it. 
+        self_epoch: 6 --you can ignore it since we do not test it. 
+        scale_poisoning: False --you can ignore it since we do not test it. 
+        scale_para: 3.0 --you can ignore it since we do not test it. 
+        pgd_poisoning: False --you can ignore it since we do not test it. 
+        mean: [0.4914, 0.4822, 0.4465] --normalizations used in backdoor attacks (different dataset have different settings.)
+        std: [0.2023, 0.1994, 0.2010]
+
+
+

benchmark/FedHPOBench/fedhpobench/utils/cost_model.py

@@ -81,7 +81,7 @@ def raw_cost(**kwargs):
 
 def get_info(cfg, configuration, fidelity, data):
     cfg = merge_cfg(cfg, configuration, fidelity)
-    model = get_model(cfg.model, list(data.values())[0])
+    model = get_model(cfg, list(data.values())[0])
     model_size = sum([param.nelement() for param in model.parameters()])
     return cfg, model_size
 

benchmark/pFL-Bench/README.md

@@ -2,7 +2,8 @@
 The **pFL-Bench** is a comprehensive benchmark for personalized Federated Learning (pFL), which contains more than 10 diverse datasets, 20 competitive pFL baselines, and systematic evaluation with highlighted benefits and potential of pFL. See more details in our [paper](https://arxiv.org/abs/2206.03655).
 
 
-This repository includes the experimental data, environments, scripts and codes of **pFL-Bench**. We welcome contributions of new pFL methods and datasets to keep pFL-Bench up-to-date and to evolve it! See more details about contribution [here](https://github.com/alibaba/FederatedScope#contributing).
+This repository mainly includes the experimental data, environments, scripts and codes of **pFL-Bench**. We welcome contributions of new pFL methods and datasets to keep pFL-Bench up-to-date and to evolve it! See more details about contribution [here](https://github.com/alibaba/FederatedScope#contributing). 
+Recently, our new proposed method for efficient pFL, [pFedGate](https://arxiv.org/abs/2305.02776) has been accepted to ICML'23. We provide its initial implementation [here](https://github.com/yxdyc/pFedGate) and will add it and more efficient pFL methods into our benchmark.
 
 **NOTICE:** We are working on seamlessly and consistently fusing the new features in pFL-Bench into the *FederatedScope*. However, since the underling package *FederatedScope* is still being continuously and actively updated, the results can be a little different to the ones in our paper.
 To fully reproduce the experimental results reported in the paper, please use the code versioned by this [branch](https://github.com/alibaba/FederatedScope/tree/Feature/pfl_bench) on which the experiments were conducted at the time.
@@ -111,3 +112,22 @@ wandb login --host=http://xx.xx.xx.xx:8080/
 ```
 
 3. connect the machine and develop your pFL algorithm
+
+
+# License
+Our codes were released with Apache-2.0 License. Please kindly cite our papers (and the respective papers of the methods used) if our work is useful for you:
+```
+@inproceedings{chen2022pflbench,
+  title={p{FL}-Bench: A Comprehensive Benchmark for Personalized Federated Learning},
+  author={Daoyuan Chen and Dawei Gao and Weirui Kuang and Yaliang Li and Bolin Ding},
+  booktitle={Thirty-sixth Conference on Neural Information Processing Systems Datasets and Benchmarks Track},
+  year={2022},
+}
+
+@inproceedings{chen2023pFedGate,
+  title={Efficient Personalized Federated Learning via Sparse Model-Adaptation},
+  author={Daoyuan Chen and Liuyi Yao and Dawei Gao and Bolin Ding and Yaliang Li},
+  booktitle={International Conference on Machine Learning},
+  year={2023},
+}
+```

configs/distributed/Phi-3-mini-128k-instruct/client_1_ds_1c_200r_30ls.yaml

@@ -0,0 +1,53 @@
+use_gpu: True
+device: 0
+expname_tag: "ds_1c_200r_30ls_dist"
+early_stop:
+  patience: 0
+federate:
+  mode: "distributed"
+  client_num: 1
+  total_round_num: 200
+  save_to: "models/distributed/Phi-3-mini-128k-instruct/ds_1c_200r_30ls.ckpt"
+  make_global_eval: False
+data:
+  root: data/
+  type: 'alpaca@llm'
+  splits: [0.98,0.01,0.01]
+  splitter: 'iid'
+distribute:
+  use: True
+  server_host: '192.168.24.120'
+  server_port: 11004
+  client_host: '192.168.24.115'
+  client_port: 50052
+  role: 'client'
+  data_idx: 1
+  grpc_max_send_message_length: 1048576000
+  grpc_max_receive_message_length: 1048576000
+llm:
+  tok_len: 1000
+  chat:
+    max_len: 2000
+  adapter:
+    use: True
+    args: [ { 'adapter_package': 'peft', 'adapter_method': 'lora', 'r': 8, 'lora_alpha': 16, 'lora_dropout': 0.05, 'target_modules': [ "q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head" ] } ]
+  deepspeed:
+    use: True
+    ds_config: 'federatedscope/llm/baseline/deepspeed/ds_config_4bs.json'
+dataloader:
+  batch_size: 1
+model:
+  type: 'microsoft/Phi-3-mini-128k-instruct@huggingface_llm'
+train:
+  local_update_steps: 30
+  batch_or_epoch: batch
+  optimizer:
+    lr: 0.0003
+    weight_decay: 0.0
+criterion:
+  type: CrossEntropyLoss
+trainer:
+  type: llmtrainer
+eval:
+  freq: 50
+  metrics: ['loss']