remove fairseq cli and modules from examples (bytedance#234)

* remove fairseq cli and modules from examples

* modify all cli import paths

* add cli for lightseq-deepspeed, delete useless __init__.py

* fix bug for test_ls_ops (bert encoder layer)

* format files

* remove examples from build

* remove examples cpp from cmakelist

* update version to 2.2.1

* Update CMakeLists.txt

recover cpp example build

Co-authored-by: Ying Xiong <[email protected]>

Author: godweiyang

MANIFEST.in

@@ -3,3 +3,4 @@ global-include *.cu *.cpp *.cc *.cuh *.h *.ldscript *.proto *.cmake
 prune dist
 prune build
 prune tests
+prune examples

examples/inference/__init__.py

@@ -8,8 +8,8 @@
 from fairseq import tasks, distributed_utils
 from fairseq.logging import metrics
 
-from examples.training.deepspeed.ds_fairseq_data import BatchIterator
-from examples.training.deepspeed.ds_fairseq_argument import gen_ds_fairseq_arg
+from ds_fairseq_data import BatchIterator
+from ds_fairseq_argument import gen_ds_fairseq_arg
 
 
 best_bleu = 0.0

examples/inference/python/__init__.py

@@ -9,9 +9,8 @@ if [ ! -d "/tmp/wmt14_en_de" ]; then
     tar -zxvf /tmp/databin_wmt14_en_de.tar.gz -C /tmp && rm /tmp/databin_wmt14_en_de.tar.gz
 fi
 
-deepspeed ${THIS_DIR}/ds_fairseq.py \
+lightseq-deepspeed ${THIS_DIR}/ds_fairseq.py \
     /tmp/wmt14_en_de/ \
-    --user-dir  ${THIS_DIR}/../fairseq/fs_modules \
     --arch ls_transformer_wmt_en_de_big_t2t --share-decoder-input-output-embed \
     --optimizer ls_adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \
     --lr 5e-4 --lr-scheduler inverse_sqrt --warmup-updates 4000 \

examples/inference/python/export/__init__.py

@@ -13,7 +13,7 @@ sh examples/training/fairseq/ls_fairseq_wmt14en2de.sh
 ```
 
 Or you can use LightSeq modules like `--arch ls_transformer_wmt_en_de_big_t2t`,
-by adding `--user-dir=${LIGHTSEQ_DIR}/examples/training/fairseq/fs_modules`
+by adding `--user-dir=${LIGHTSEQ_DIR}/lightseq/training/cli/fs_modules`
 to `fairseq-train`.
 
 This script firstly download the dataset and then run native Fairseq