From 09b6274a1fdd3b514c1e0948bc95860c0e809427 Mon Sep 17 00:00:00 2001 From: Mu Yang Date: Sun, 9 Apr 2023 13:30:07 +0800 Subject: [PATCH 1/3] =?UTF-8?q?chore:=20=F0=9F=A4=96=20bump=20to=200.3.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.rst | 2 +- ckip_transformers/__init__.py | 4 ++-- ckip_transformers/nlp/__init__.py | 2 +- ckip_transformers/nlp/driver.py | 2 +- ckip_transformers/nlp/util.py | 2 +- setup.py | 2 +- test/script/nlp/_base.py | 2 +- test/script/nlp/run.py | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 905b004..e692858 100644 --- a/README.rst +++ b/README.rst @@ -472,7 +472,7 @@ License |GPL-3.0| -Copyright (c) 2021 `CKIP Lab `__ under the `GPL-3.0 License `__. +Copyright (c) 2023 `CKIP Lab `__ under the `GPL-3.0 License `__. .. |GPL-3.0| image:: https://www.gnu.org/graphics/gplv3-with-text-136x68.png :target: https://www.gnu.org/licenses/gpl-3.0.html diff --git a/ckip_transformers/__init__.py b/ckip_transformers/__init__.py index 79ca929..7465466 100644 --- a/ckip_transformers/__init__.py +++ b/ckip_transformers/__init__.py @@ -7,10 +7,10 @@ __author_name__ = "Mu Yang" __author_email__ = "emfomy@gmail.com" -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __title__ = "CKIP Transformers" -__version__ = "0.3.2" +__version__ = "0.3.3" __description__ = "CKIP Transformers" __license__ = "GPL-3.0" diff --git a/ckip_transformers/nlp/__init__.py b/ckip_transformers/nlp/__init__.py index 5061fd5..42e5427 100644 --- a/ckip_transformers/nlp/__init__.py +++ b/ckip_transformers/nlp/__init__.py @@ -6,7 +6,7 @@ """ __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" from .driver import ( diff --git a/ckip_transformers/nlp/driver.py b/ckip_transformers/nlp/driver.py index 623c255..cb2e41f 100644 --- a/ckip_transformers/nlp/driver.py +++ b/ckip_transformers/nlp/driver.py @@ -6,7 +6,7 @@ """ __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" from typing import ( diff --git a/ckip_transformers/nlp/util.py b/ckip_transformers/nlp/util.py index 7ec313a..98d0b4e 100644 --- a/ckip_transformers/nlp/util.py +++ b/ckip_transformers/nlp/util.py @@ -6,7 +6,7 @@ """ __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" diff --git a/setup.py b/setup.py index 9b6452c..62678c0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ # -*- coding:utf-8 -*- __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" from setuptools import setup, find_namespace_packages diff --git a/test/script/nlp/_base.py b/test/script/nlp/_base.py index 4143707..9c56f33 100644 --- a/test/script/nlp/_base.py +++ b/test/script/nlp/_base.py @@ -2,7 +2,7 @@ # -*- coding:utf-8 -*- __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" from ckip_transformers.nlp import * diff --git a/test/script/nlp/run.py b/test/script/nlp/run.py index 3f805b4..2b5c3ff 100644 --- a/test/script/nlp/run.py +++ b/test/script/nlp/run.py @@ -2,7 +2,7 @@ # -*- coding:utf-8 -*- __author__ = "Mu Yang " -__copyright__ = "2021 CKIP Lab" +__copyright__ = "2023 CKIP Lab" __license__ = "GPL-3.0" from _base import * From 3b99a5f6f5fbcacfb24a6296136a8c06db32aa7b Mon Sep 17 00:00:00 2001 From: Mu Yang Date: Sun, 9 Apr 2023 13:30:28 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20disable=20pin=20memo?= =?UTF-8?q?ry=20on=20CPU=20device?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ckip_transformers/nlp/driver.py | 12 ++++++------ ckip_transformers/nlp/util.py | 8 ++++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ckip_transformers/nlp/driver.py b/ckip_transformers/nlp/driver.py index cb2e41f..75b2c0d 100644 --- a/ckip_transformers/nlp/driver.py +++ b/ckip_transformers/nlp/driver.py @@ -77,8 +77,8 @@ def __call__( show_progress : ``int``, *optional*, defaults to True Show progress bar. pin_memory : ``bool``, *optional*, defaults to True - Pin memory in order to accelerate the speed of data transfer to the GPU. This option is - incompatible with multiprocessing. + Pin memory in order to accelerate the speed of data transfer to the GPU. This option is incompatible with + multiprocessing. Disabled on CPU device. Returns ------- @@ -177,8 +177,8 @@ def __call__( show_progress : ``int``, *optional*, defaults to True Show progress bar. pin_memory : ``bool``, *optional*, defaults to True - Pin memory in order to accelerate the speed of data transfer to the GPU. This option is - incompatible with multiprocessing. + Pin memory in order to accelerate the speed of data transfer to the GPU. This option is incompatible with + multiprocessing. Disabled on CPU device. Returns ------- @@ -267,8 +267,8 @@ def __call__( show_progress : ``int``, *optional*, defaults to True Show progress bar. pin_memory : ``bool``, *optional*, defaults to True - Pin memory in order to accelerate the speed of data transfer to the GPU. This option is - incompatible with multiprocessing. + Pin memory in order to accelerate the speed of data transfer to the GPU. This option is incompatible with + multiprocessing. Disabled on CPU device. Returns ------- diff --git a/ckip_transformers/nlp/util.py b/ckip_transformers/nlp/util.py index 98d0b4e..257c41a 100644 --- a/ckip_transformers/nlp/util.py +++ b/ckip_transformers/nlp/util.py @@ -123,10 +123,14 @@ def __call__( show_progress : ``bool``, *optional*, defaults to True Show progress bar. pin_memory : ``bool``, *optional*, defaults to True - Pin memory in order to accelerate the speed of data transfer to the GPU. This option is - incompatible with multiprocessing. + Pin memory in order to accelerate the speed of data transfer to the GPU. This option is incompatible with + multiprocessing. Disabled on CPU device. """ + # Disable pin memory on CPU device + if self.device.type == "cpu": + pin_memory = False + # Check max length model_max_length = self.tokenizer.model_max_length - 2 # Add [CLS] and [SEP] if max_length: assert max_length < model_max_length, ( From 8863fc94fefb4c6ac4da568c7a71368496dd457b Mon Sep 17 00:00:00 2001 From: Mu Yang Date: Sun, 9 Apr 2023 13:35:58 +0800 Subject: [PATCH 3/3] =?UTF-8?q?docs:=20=E2=9C=8F=EF=B8=8F=20table=20of=20t?= =?UTF-8?q?ags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/index.rst | 5 ++++ docs/main/tag.rst | 15 ++++++++++ docs/main/tag/ner.csv | 19 +++++++++++++ docs/main/tag/pos.csv | 65 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 docs/main/tag.rst create mode 100644 docs/main/tag/ner.csv create mode 100644 docs/main/tag/pos.csv diff --git a/docs/index.rst b/docs/index.rst index 2540cfb..1ae51fe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,11 @@ CKIP Transformers main/readme +.. toctree:: + :caption: Tables of Tags + + main/tag + .. toctree:: :caption: Contents diff --git a/docs/main/tag.rst b/docs/main/tag.rst new file mode 100644 index 0000000..8df2c90 --- /dev/null +++ b/docs/main/tag.rst @@ -0,0 +1,15 @@ +Part-of-Speech Tags +------------------- + +.. csv-table:: + :file: ./tag/pos.csv + :widths: 50 50 + :header-rows: 1 + +Named Entity Recognition Tags +----------------------------- + +.. csv-table:: + :file: ./tag/ner.csv + :widths: 50 50 + :header-rows: 1 diff --git a/docs/main/tag/ner.csv b/docs/main/tag/ner.csv new file mode 100644 index 0000000..206eea8 --- /dev/null +++ b/docs/main/tag/ner.csv @@ -0,0 +1,19 @@ +Tag,Description +CARDINAL,數字 +DATE,日期 +EVENT,事件 +FAC,設施 +GPE,行政區 +LANGUAGE,語言 +LAW,法律 +LOC,地理區 +MONEY,金錢 +NORP,民族、宗教、政治團體 +ORDINAL,序數 +ORG,組織 +PERCENT,百分比率 +PERSON,人物 +PRODUCT,產品 +QUANTITY,數量 +TIME,時間 +WORK_OF_ART,作品 diff --git a/docs/main/tag/pos.csv b/docs/main/tag/pos.csv new file mode 100644 index 0000000..787a455 --- /dev/null +++ b/docs/main/tag/pos.csv @@ -0,0 +1,65 @@ +Tag,Description +A,非謂形容詞 +Caa,對等連接詞 +Cab,連接詞，如：等等 +Cba,連接詞，如：的話 +Cbb,關聯連接詞 +D,副詞 +Da,數量副詞 +Dfa,動詞前程度副詞 +Dfb,動詞後程度副詞 +Di,時態標記 +Dk,句副詞 +DM,定量式 +I,感嘆詞 +Na,普通名詞 +Nb,專有名詞 +Nc,地方詞 +Ncd,位置詞 +Nd,時間詞 +Nep,指代定詞 +Neqa,數量定詞 +Neqb,後置數量定詞 +Nes,特指定詞 +Neu,數詞定詞 +Nf,量詞 +Ng,後置詞 +Nh,代名詞 +Nv,名物化動詞 +P,介詞 +T,語助詞 +VA,動作不及物動詞 +VAC,動作使動動詞 +VB,動作類及物動詞 +VC,動作及物動詞 +VCL,動作接地方賓語動詞 +VD,雙賓動詞 +VF,動作謂賓動詞 +VE,動作句賓動詞 +VG,分類動詞 +VH,狀態不及物動詞 +VHC,狀態使動動詞 +VI,狀態類及物動詞 +VJ,狀態及物動詞 +VK,狀態句賓動詞 +VL,狀態謂賓動詞 +V_2,有 + +DE,的之得地 +SHI,是 +FW,外文 + +COLONCATEGORY,冒號 +COMMACATEGORY,逗號 +DASHCATEGORY,破折號 +DOTCATEGORY,點號 +ETCCATEGORY,刪節號 +EXCLAMATIONCATEGORY,驚嘆號 +PARENTHESISCATEGORY,括號 +PAUSECATEGORY,頓號 +PERIODCATEGORY,句號 +QUESTIONCATEGORY,問號 +SEMICOLONCATEGORY,分號 +SPCHANGECATEGORY,雙直線 + +WHITESPACE,空白