From 205acfcfd8821aee2eed560c4c1e8fe72c380338 Mon Sep 17 00:00:00 2001 From: clement10601 Date: Wed, 23 May 2018 15:08:08 +0800 Subject: [PATCH 01/15] update --- req.txt | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 req.txt diff --git a/req.txt b/req.txt new file mode 100644 index 000000000..18098d8d2 --- /dev/null +++ b/req.txt @@ -0,0 +1,115 @@ +absl-py==0.1.12 +amqp==2.2.2 +args==0.1.0 +asn1crypto==0.24.0 +astor==0.6.2 +billiard==3.5.0.3 +bleach==1.5.0 +celery==4.1.0 +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +clint==0.5.1 +conda==4.4.10 +crontab==0.22.0 +cryptography==2.2.1 +cycler==0.10.0 +Cython==0.28.1 +decorator==4.2.1 +dev==0.4.0 +docopt==0.6.2 +entrypoints==0.2.3 +enum34==1.1.6 +et-xmlfile==1.0.1 +Flask==0.12.2 +Flask-Script==2.0.6 +futures==3.1.1 +gast==0.2.0 +gevent==1.2.2 +GPUtil==1.2.3 +greenlet==0.4.13 +grpcio==1.10.0 +gunicorn==19.7.1 +h5py==2.7.1 +html5lib==0.9999999 +idna==2.6 +ipykernel==4.8.2 +ipython==6.2.1 +ipython-genutils==0.2.0 +ipywidgets==7.1.2 +itsdangerous==0.24 +jdcal==1.3 +jedi==0.11.1 +Jinja2==2.10 +jsonschema==2.6.0 +jupyter-client==5.2.3 +jupyter-core==4.4.0 +Keras==2.1.5 +kiwisolver==1.0.1 +kombu==4.1.0 +Markdown==2.6.11 +MarkupSafe==1.0 +matplotlib==2.2.2 +mistune==0.8.3 +nbconvert==5.3.1 +nbformat==4.4.0 +networkx==2.1 +notebook==5.4.1 +numpy==1.14.2 +odfpy==1.3.6 +opencv-python==3.4.0.12 +openpyxl==2.5.1 +pandocfilters==1.4.2 +parso==0.1.1 +pbr==4.0.2 +pexpect==4.4.0 +pickleshare==0.7.4 +Pillow==5.0.0 +prompt-toolkit==1.0.15 +protobuf==3.5.2.post1 +ptyprocess==0.5.2 +pycocotools==2.0 +pycosat==0.6.3 +pycparser==2.18 +Pygments==2.2.0 +pyOpenSSL==17.5.0 +pyparsing==2.2.0 +PySocks==1.6.8 +python-crontab==2.2.8 +python-dateutil==2.7.2 +pytz==2018.3 +PyWavelets==0.5.2 +PyYAML==3.12 +pyzmq==17.0.0 +redis==2.10.6 +requests==2.18.4 +ruamel.yaml==0.15.37 +scikit-image==0.13.1 +scikit-learn==0.19.1 +scipy==1.0.1 +Send2Trash==1.5.0 +simplegeneric==0.8.1 +six==1.11.0 +stevedore==1.28.0 +tablib==0.12.1 +tensorboard==1.6.0 +tensorflow==1.6.0 +tensorflow-tensorboard==1.5.1 +termcolor==1.1.0 +terminado==0.8.1 +testpath==0.3.1 +tornado==5.0.1 +traitlets==4.3.2 +unicodecsv==0.14.1 +urllib3==1.22 +vine==1.1.4 +virtualenv==15.2.0 +virtualenv-clone==0.3.0 +virtualenvwrapper==4.8.2 +wcwidth==0.1.7 +webencodings==0.5.1 +Werkzeug==0.14.1 +widgetsnbextension==3.1.4 +xlrd==1.1.0 +xlwt==1.3.0 From 81051b2f5540103ebf5fe1efb44a0edd21f29518 Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 15:09:58 +0800 Subject: [PATCH 02/15] add req --- req.txt | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 req.txt diff --git a/req.txt b/req.txt new file mode 100644 index 000000000..18098d8d2 --- /dev/null +++ b/req.txt @@ -0,0 +1,115 @@ +absl-py==0.1.12 +amqp==2.2.2 +args==0.1.0 +asn1crypto==0.24.0 +astor==0.6.2 +billiard==3.5.0.3 +bleach==1.5.0 +celery==4.1.0 +certifi==2018.1.18 +cffi==1.11.5 +chardet==3.0.4 +click==6.7 +clint==0.5.1 +conda==4.4.10 +crontab==0.22.0 +cryptography==2.2.1 +cycler==0.10.0 +Cython==0.28.1 +decorator==4.2.1 +dev==0.4.0 +docopt==0.6.2 +entrypoints==0.2.3 +enum34==1.1.6 +et-xmlfile==1.0.1 +Flask==0.12.2 +Flask-Script==2.0.6 +futures==3.1.1 +gast==0.2.0 +gevent==1.2.2 +GPUtil==1.2.3 +greenlet==0.4.13 +grpcio==1.10.0 +gunicorn==19.7.1 +h5py==2.7.1 +html5lib==0.9999999 +idna==2.6 +ipykernel==4.8.2 +ipython==6.2.1 +ipython-genutils==0.2.0 +ipywidgets==7.1.2 +itsdangerous==0.24 +jdcal==1.3 +jedi==0.11.1 +Jinja2==2.10 +jsonschema==2.6.0 +jupyter-client==5.2.3 +jupyter-core==4.4.0 +Keras==2.1.5 +kiwisolver==1.0.1 +kombu==4.1.0 +Markdown==2.6.11 +MarkupSafe==1.0 +matplotlib==2.2.2 +mistune==0.8.3 +nbconvert==5.3.1 +nbformat==4.4.0 +networkx==2.1 +notebook==5.4.1 +numpy==1.14.2 +odfpy==1.3.6 +opencv-python==3.4.0.12 +openpyxl==2.5.1 +pandocfilters==1.4.2 +parso==0.1.1 +pbr==4.0.2 +pexpect==4.4.0 +pickleshare==0.7.4 +Pillow==5.0.0 +prompt-toolkit==1.0.15 +protobuf==3.5.2.post1 +ptyprocess==0.5.2 +pycocotools==2.0 +pycosat==0.6.3 +pycparser==2.18 +Pygments==2.2.0 +pyOpenSSL==17.5.0 +pyparsing==2.2.0 +PySocks==1.6.8 +python-crontab==2.2.8 +python-dateutil==2.7.2 +pytz==2018.3 +PyWavelets==0.5.2 +PyYAML==3.12 +pyzmq==17.0.0 +redis==2.10.6 +requests==2.18.4 +ruamel.yaml==0.15.37 +scikit-image==0.13.1 +scikit-learn==0.19.1 +scipy==1.0.1 +Send2Trash==1.5.0 +simplegeneric==0.8.1 +six==1.11.0 +stevedore==1.28.0 +tablib==0.12.1 +tensorboard==1.6.0 +tensorflow==1.6.0 +tensorflow-tensorboard==1.5.1 +termcolor==1.1.0 +terminado==0.8.1 +testpath==0.3.1 +tornado==5.0.1 +traitlets==4.3.2 +unicodecsv==0.14.1 +urllib3==1.22 +vine==1.1.4 +virtualenv==15.2.0 +virtualenv-clone==0.3.0 +virtualenvwrapper==4.8.2 +wcwidth==0.1.7 +webencodings==0.5.1 +Werkzeug==0.14.1 +widgetsnbextension==3.1.4 +xlrd==1.1.0 +xlwt==1.3.0 From 2c232cb933f8414cf11deb6d4ebe7be4dd376b7e Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 15:25:57 +0800 Subject: [PATCH 03/15] add yolo3 for realtime cam view --- req.txt | 1 - utils/__pycache__/__init__.cpython-36.pyc | Bin 0 -> 136 bytes utils/__pycache__/bbox.cpython-36.pyc | Bin 0 -> 2619 bytes utils/__pycache__/colors.cpython-36.pyc | Bin 0 -> 1685 bytes utils/__pycache__/weightreader.cpython-36.pyc | Bin 0 -> 2368 bytes utils/weightreader.py | 69 ++++ yolo3_cam.py | 352 ++++++++++++++++++ 7 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 utils/__pycache__/__init__.cpython-36.pyc create mode 100644 utils/__pycache__/bbox.cpython-36.pyc create mode 100644 utils/__pycache__/colors.cpython-36.pyc create mode 100644 utils/__pycache__/weightreader.cpython-36.pyc create mode 100644 utils/weightreader.py create mode 100644 yolo3_cam.py diff --git a/req.txt b/req.txt index 18098d8d2..485e8747c 100644 --- a/req.txt +++ b/req.txt @@ -11,7 +11,6 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 clint==0.5.1 -conda==4.4.10 crontab==0.22.0 cryptography==2.2.1 cycler==0.10.0 diff --git a/utils/__pycache__/__init__.cpython-36.pyc b/utils/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e325baa4d10a1b1be36e38eb3ba0a2cafe79d18 GIT binary patch literal 136 zcmXr!<>iupeJzRs2p)q77+?f49Dul(1xTbY1T$zd`mJOr0tq9CU#9w@#i>Qb`pG$| zxv6<2`T<4xS*gh-#roN)MTy0_mH9dO#`>itnK{M!@$s2?nI-Y@dIgoYIBatBQ%ZAE L?LfvA12F>tQq&;w literal 0 HcmV?d00001 diff --git a/utils/__pycache__/bbox.cpython-36.pyc b/utils/__pycache__/bbox.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d39c1a6980a0496e90ec16f1791691502076be45 GIT binary patch literal 2619 zcmaJ?OK%)S5bo}IuXp{*Cb3NfIRwO_5GxOfr$`h?2of#ffW#1Yh+yj)?*@>ZeNvZ?VSw45zY<83%1}9kk;-Y^6!6S~UOSJF?*&%)~}Nbo-3SO{~6#|qO7N{s}P^)l2?ZO3hBuQOK zQcp4%ZEq+#WcD{c<)GK_wLR{42io2@bli`|)Cc7)?X{99%i>HsNwgg&+R0j}iW{D` zvpCsNE)Gy+nDU7PL@Gq!3HFSs1+eEA*Y9Pq%GO&++>ZzO`ld>IaVyW(cViW0@9zV$ z%j=`On`G4yMG(f_@)zJ}2TLyyy6|FciyT28@gt<(#Gqh3 zCdIB$D>?50k)AazvBNwtu}VxzIBZqL4lM>l?MA8%9L!}JT+L(N+^8gERq1pC%?S)M z!h`V&pBD$qNA^)%m?f__@GQ0*8MuZXOzVj$Tg5Eg`x+B1Rl^=*ma3!of3_KatzyxU zZHuqkR=QMdY&2W~eh~HJFw~VW?5A>+P`?_64@Xf_O4Jf5C%x(<5%TddLa-l+!%*4-8UgQj6j?65?3fwkW z!^np0+P`Er$xNt=4Rx5B79ty$t8@o>tR6*47#-_xYcalvb_wk=+Lc!>qn8kihx^h; zu}Y%hTQE3V24URizCh!)SOm9)I*Zw(Yf>%{7Ur5y(ZLlC9o!ZLhpQ2fZ862wjRbkx zvyB8mP!AFd8E)0t1i5Wum#@$nG_qC@<`ZY)(kXNrB-n*kO1y~=_}n@xyuz=u)2skE zlOPAUi2vXXihT~R8=K2qHfd}UnsQk?J4qVljf%FBm=_Jbq~{DA8B#dhJ3i>>;7Fpy z9x&~o?E%-$CGCs`-E{COv_2(#*iA^WV|{ar`-x!3ax z7c`IooKulF5hqlp_9Q3*8c{p!iB)o4Wipq~^{UW^$^z#(y?HqQ*r>p)tgf(fwy408 z8r+C=|1D{Ywk_x7!V@v6!kUGG*4Jpy4xPP-wzMPi<-(hCS&@rRQ1`Iz1lCpC{BVBh ziDhQ%n00CY3%#)JCktSjPn0E zf7>KxdwuXT|HZj;bw>W3(Q`0cBSLQ04r1x@CDc_%=KI6A;b^PdZ#S)Bevt?w%G6pN z2gc|6XWR;t=fe-U6p{=IVcN_3 zN;qkMr<)|3X|k`q;mGJ{)BssOf^iOV3KiQmH7C=Rbh{Vk9TjIC?K~WHV;F}OIxzAO z_Lmipcbfs>6J}(sv)7e*N4uT4+wSDrRdM?@`CE}H+6#@oPheI?_%lAwZM=lb;6C&D zdo*HIypCs(+sphE=o#@YxXT5`Hm`zLg}R5m4SpA|jWZbahx;WcBF)rwNVIGGohiD% pDJJE!MK6TedhBg6CCSuh|I@OFeMXhQJTUZF`%pK(>M!`Ue*o;9O)dZc literal 0 HcmV?d00001 diff --git a/utils/__pycache__/colors.cpython-36.pyc b/utils/__pycache__/colors.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec9045f86b95574c4da37c2db7ace19f8857d784 GIT binary patch literal 1685 zcmd6nJ8u&~5P4$s=YaK4Ln*GLXQ z6hKFTsGx#|0*Q)>j*1E@3WVHHAPS3)3emvK-s;Ftz?Q$+dC%-wx6t2Tn*8y36?z%_ z#T!-SVBkDps|<4Bf>(Euc_Te2fQz~ZJyxR6=tU#R(`$4WN=EK5_P(r6 zzpdxNfQ=hOIb{7ivp5fi(cXvksQ!qxqXwg>@3QvYS^OS5-(EY93q2S|@kNwlHh!O# z9_-KNP51L)9OEeCfX&;Baw4nmLEiYs?fe5)P8ylZ>hi4rl&yCV<)O@fIO{u!cABq% za+mEhlEod#;-*d9QL7KzJzY3r^EpZvX6!r%jNF;knab*vF>cn{UFfy(9?Y2hW0~HM z`jCw~Wak>Qa@NX;tiB67Fy4bv(`U}w=WQJAchvYx#=p`@-xB>-XAxUWGws}WZZSEkoh>%TR+!F#v+i!O`)GF<(|Jx+P;C@M;nzlGkL?L`4bBlN#TU0z@s01*C3O;Y&%ZgD_V7t>~zH2qq**_Qq-Ap_V<3=Lx6wHjOli!eGtUHBTBAbCOJDt2)>=lzmO!>XwUspmZzZ!`qPUc^g$-1a& zWiP0W4`;z|=`g|&dQhse(eCu(J**j(dMhf#ihWKoH{i$;p1YHXe{*%8lWmPKLYO9u z5grm=6Cwh={<;OiQ^Hq5AK?w*E#WfZ31NZop74usgK&@VjPQlfCR`+(Bb*|9BGd>! z3GWD!@PqK3@Q5%+cu5e14}_0|6NoG2oLN+QsF;h~27IZrPVr1LfmS5Wnj3n7rs81H I`Jcak0jE5DZ~y=R literal 0 HcmV?d00001 diff --git a/utils/__pycache__/weightreader.cpython-36.pyc b/utils/__pycache__/weightreader.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c560b06d54107acce1f28877164d135116881c6b GIT binary patch literal 2368 zcmb^y%Z?jGu)94kkH;IwBqSRmWJo|Vk6o0UkPwPUB7_uHAt59gh(=>~+ZlTvPEUJ- zt=W^q4a6S!fQTD>0-w@XF8CKXQPs20C`YE%Rn=Y9)m7D1HP_p1f8)IU{bQeyzsZTO z0rU-+HotE9+v^Nxt8i+keE1KkdVEbhjfC;1o-1rKKT<7}3iRiEhwaCB=} zbcpvIehi2kFw2hs2!dXRj8o2D5b(UgEx=Z7b7*syd$4wR6YTbN z@K7Y{NM&7s7*1`eVC_r^VCw**R(C}y$^F9{C^1sj>}Mlo?K#Loq0Zshf{4yh?aJW(Wn$kvwWugku0*od^8eLx8gL7o|~wa zsG=0W((~_QIaPriULqn zz6pTPfS$v_dNg1@>##0!=<@u@bl5sPb(9wvH>@|47id&PCXg8y07ppEdVSIC|HL*i z{&c6$bp`ptQr_4H*Ix?&cg6Nwu z>`!>lefs0?3Yx)+-U9l`Z+mQ?E=CodKv^hA-W3?V!o-CnK3+qrG2WiK@}-eo&7MUtuV=2oH@-R=AoxC`GxZjX{ z6!`$bhXDE=Z9~P4g>H_8iqd!iHQa!T*Y->%xssb8q^+4OpcSLUfmzxvpT;vG-$!-} z8o?2xkwVXKr7(PLrAetweM9XEE_*2Ne;227aa+nl_SdZNSvjP9YS$SXeqKxmg* zFgwhnU3QN7)C}keU3!ULVwVl?5;d51=yLN^L7vn(#23I1G4=~nlGt@Qbcd*Za<*e$ zt4~d10E9i^Ra0#Up-|>H>LGO5J$0*L@AO^lZh4%EDAIluK^W&L(m@nGo5yL*kyr5o zd@Fei!EFS`p)u4I+{3>f&JuqN9`(AO<-KEe`)_D3mg89r&!|~ZYCE52vqhaBf$70p z=9x-~=1R;zi5*PZ2pj}10Np()yOjw!7SjBNn#euSWU`@@7&mzy_Z!3AYsQY(nXSFg VYQI0nsF#@kU^|Q+JXU~I<3Ho8P0auR literal 0 HcmV?d00001 diff --git a/utils/weightreader.py b/utils/weightreader.py new file mode 100644 index 000000000..02c4de8c3 --- /dev/null +++ b/utils/weightreader.py @@ -0,0 +1,69 @@ +import argparse +import os +import numpy as np +from keras.layers import Conv2D, Input, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D +from keras.layers.merge import add, concatenate +from keras.models import Model +import struct +import cv2 + + +class WeightReader: + def __init__(self, weight_file): + with open(weight_file, 'rb') as w_f: + major, = struct.unpack('i', w_f.read(4)) + minor, = struct.unpack('i', w_f.read(4)) + revision, = struct.unpack('i', w_f.read(4)) + + if (major*10 + minor) >= 2 and major < 1000 and minor < 1000: + w_f.read(8) + else: + w_f.read(4) + + transpose = (major > 1000) or (minor > 1000) + + binary = w_f.read() + + self.offset = 0 + self.all_weights = np.frombuffer(binary, dtype='float32') + + def read_bytes(self, size): + self.offset = self.offset + size + return self.all_weights[self.offset-size:self.offset] + + def load_weights(self, model): + for i in range(106): + try: + conv_layer = model.get_layer('conv_' + str(i)) + print("loading weights of convolution #" + str(i)) + + if i not in [81, 93, 105]: + norm_layer = model.get_layer('bnorm_' + str(i)) + + size = np.prod(norm_layer.get_weights()[0].shape) + + beta = self.read_bytes(size) # bias + gamma = self.read_bytes(size) # scale + mean = self.read_bytes(size) # mean + var = self.read_bytes(size) # variance + + weights = norm_layer.set_weights([gamma, beta, mean, var]) + + if len(conv_layer.get_weights()) > 1: + bias = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape)) + kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape)) + + kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape))) + kernel = kernel.transpose([2,3,1,0]) + conv_layer.set_weights([kernel, bias]) + else: + kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape)) + kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape))) + kernel = kernel.transpose([2,3,1,0]) + conv_layer.set_weights([kernel]) + except ValueError: + print("no convolution #" + str(i)) + + def reset(self): + self.offset = 0 + diff --git a/yolo3_cam.py b/yolo3_cam.py new file mode 100644 index 000000000..e5b77a1a0 --- /dev/null +++ b/yolo3_cam.py @@ -0,0 +1,352 @@ +import argparse +import os +import numpy as np +from keras.layers import Conv2D, Input, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D +from keras.layers.merge import add, concatenate +from keras.models import Model +import struct +import cv2 +from utils.weightreader import WeightReader +from utils.bbox import BoundBox + +np.set_printoptions(threshold=np.nan) +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="0" + +argparser = argparse.ArgumentParser( + description='test yolov3 network with coco weights') + +argparser.add_argument( + '-w', + '--weights', + help='path to weights file') + +argparser.add_argument( + '-i', + '--image', + help='path to image file') + +def _conv_block(inp, convs, skip=True): + x = inp + count = 0 + + for conv in convs: + if count == (len(convs) - 2) and skip: + skip_connection = x + count += 1 + + if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top + x = Conv2D(conv['filter'], + conv['kernel'], + strides=conv['stride'], + padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top + name='conv_' + str(conv['layer_idx']), + use_bias=False if conv['bnorm'] else True)(x) + if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) + if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) + + return add([skip_connection, x]) if skip else x + +def _interval_overlap(interval_a, interval_b): + x1, x2 = interval_a + x3, x4 = interval_b + + if x3 < x1: + if x4 < x1: + return 0 + else: + return min(x2,x4) - x1 + else: + if x2 < x3: + return 0 + else: + return min(x2,x4) - x3 + +def _sigmoid(x): + return 1. / (1. + np.exp(-x)) + +def bbox_iou(box1, box2): + intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) + intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) + + intersect = intersect_w * intersect_h + + w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin + w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin + + union = w1*h1 + w2*h2 - intersect + + return float(intersect) / union + +def make_yolov3_model(): + input_image = Input(shape=(None, None, 3)) + + # Layer 0 => 4 + x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0}, + {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1}, + {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2}, + {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}]) + + # Layer 5 => 8 + x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5}, + {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}]) + + # Layer 9 => 11 + x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}]) + + # Layer 12 => 15 + x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}]) + + # Layer 16 => 36 + for i in range(7): + x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}]) + + skip_36 = x + + # Layer 37 => 40 + x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}]) + + # Layer 41 => 61 + for i in range(7): + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}]) + + skip_61 = x + + # Layer 62 => 65 + x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}]) + + # Layer 66 => 74 + for i in range(3): + x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}]) + + # Layer 75 => 79 + x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False) + + # Layer 80 => 82 + yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False) + + # Layer 83 => 86 + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_61]) + + # Layer 87 => 91 + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False) + + # Layer 92 => 94 + yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False) + + # Layer 95 => 98 + x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}], skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_36]) + + # Layer 99 => 106 + yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False) + + model = Model(input_image, [yolo_82, yolo_94, yolo_106]) + return model + +def preprocess_input(image, net_h, net_w): + new_h, new_w, _ = image.shape + + # determine the new size of the image + if (float(net_w)/new_w) < (float(net_h)/new_h): + new_h = (new_h * net_w)/new_w + new_w = net_w + else: + new_w = (new_w * net_h)/new_h + new_h = net_h + + # resize the image to the new size + resized = cv2.resize(image[:,:,::-1]/255., (int(new_w), int(new_h))) + + # embed the image into the standard letter box + new_image = np.ones((net_h, net_w, 3)) * 0.5 + new_image[int((net_h-new_h)//2):int((net_h+new_h)//2), int((net_w-new_w)//2):int((net_w+new_w)//2), :] = resized + new_image = np.expand_dims(new_image, 0) + + return new_image + +def decode_netout(netout, anchors, obj_thresh, nms_thresh, net_h, net_w): + grid_h, grid_w = netout.shape[:2] + nb_box = 3 + netout = netout.reshape((grid_h, grid_w, nb_box, -1)) + nb_class = netout.shape[-1] - 5 + + boxes = [] + + netout[..., :2] = _sigmoid(netout[..., :2]) + netout[..., 4:] = _sigmoid(netout[..., 4:]) + netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:] + netout[..., 5:] *= netout[..., 5:] > obj_thresh + + for i in range(grid_h*grid_w): + row = i / grid_w + col = i % grid_w + + for b in range(nb_box): + # 4th element is objectness score + objectness = netout[int(row)][int(col)][b][4] + #objectness = netout[..., :4] + + if(objectness.all() <= obj_thresh): continue + + # first 4 elements are x, y, w, and h + x, y, w, h = netout[int(row)][int(col)][b][:4] + + x = (col + x) / grid_w # center position, unit: image width + y = (row + y) / grid_h # center position, unit: image height + w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width + h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height + + # last elements are class probabilities + classes = netout[int(row)][col][b][5:] + + box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes) + #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes) + + boxes.append(box) + + return boxes + +def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): + if (float(net_w)/image_w) < (float(net_h)/image_h): + new_w = net_w + new_h = (image_h*net_w)/image_w + else: + new_h = net_w + new_w = (image_w*net_h)/image_h + + for i in range(len(boxes)): + x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w + y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h + + boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) + boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) + boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) + boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) + +def do_nms(boxes, nms_thresh): + if len(boxes) > 0: + nb_class = len(boxes[0].classes) + else: + return + + for c in range(nb_class): + sorted_indices = np.argsort([-box.classes[c] for box in boxes]) + + for i in range(len(sorted_indices)): + index_i = sorted_indices[i] + + if boxes[index_i].classes[c] == 0: continue + + for j in range(i+1, len(sorted_indices)): + index_j = sorted_indices[j] + + if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: + boxes[index_j].classes[c] = 0 + +def draw_boxes(image, boxes, labels, obj_thresh): + for box in boxes: + label_str = '' + label = -1 + + for i in range(len(labels)): + if box.classes[i] > obj_thresh: + label_str += labels[i] + label = i + print(labels[i] + ': ' + str(box.classes[i]*100) + '%') + + if label >= 0: + cv2.rectangle(image, (box.xmin,box.ymin), (box.xmax,box.ymax), (0,255,0), 3) + cv2.putText(image, + label_str + ' ' + str(box.get_score()), + (box.xmin, box.ymin - 13), + cv2.FONT_HERSHEY_SIMPLEX, + 1e-3 * image.shape[0], + (0,255,0), 2) + + return image + +def _main_(args): + weights_path = args.weights + image_path = args.image + + # set some parameters + net_h, net_w = 416, 416 + obj_thresh, nms_thresh = 0.5, 0.45 + anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] + labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ + "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ + "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ + "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ + "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] + + # make the yolov3 model to predict 80 classes on COCO + yolov3 = make_yolov3_model() + + # load the weights trained on COCO into the model + weight_reader = WeightReader(weights_path) + weight_reader.load_weights(yolov3) + + # preprocess the image + image = cv2.imread(image_path) + image_h, image_w, _ = image.shape + new_image = preprocess_input(image, net_h, net_w) + + # run the prediction + yolos = yolov3.predict(new_image) + boxes = [] + + for i in range(len(yolos)): + # decode the output of the network + boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, nms_thresh, net_h, net_w) + + # correct the sizes of the bounding boxes + correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) + + # suppress non-maximal boxes + do_nms(boxes, nms_thresh) + + # draw bounding boxes on the image using labels + draw_boxes(image, boxes, labels, obj_thresh) + + # write the image with bounding boxes to file + cv2.imwrite(image_path[:-4] + '_detected' + image_path[-4:], (image).astype('uint8')) + +if __name__ == '__main__': + args = argparser.parse_args() + _main_(args) From ccb8fcbf7fefe6838dc875a603e5c7692b6b280f Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 15:27:14 +0800 Subject: [PATCH 04/15] change import path --- yolo3_cam.py | 5 -- yolo3_one_file_to_detect_them_all.py | 86 +--------------------------- 2 files changed, 2 insertions(+), 89 deletions(-) diff --git a/yolo3_cam.py b/yolo3_cam.py index e5b77a1a0..1e8960421 100644 --- a/yolo3_cam.py +++ b/yolo3_cam.py @@ -21,11 +21,6 @@ '--weights', help='path to weights file') -argparser.add_argument( - '-i', - '--image', - help='path to image file') - def _conv_block(inp, convs, skip=True): x = inp count = 0 diff --git a/yolo3_one_file_to_detect_them_all.py b/yolo3_one_file_to_detect_them_all.py index 17ca64e22..e5b77a1a0 100644 --- a/yolo3_one_file_to_detect_them_all.py +++ b/yolo3_one_file_to_detect_them_all.py @@ -6,6 +6,8 @@ from keras.models import Model import struct import cv2 +from utils.weightreader import WeightReader +from utils.bbox import BoundBox np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" @@ -24,90 +26,6 @@ '--image', help='path to image file') -class WeightReader: - def __init__(self, weight_file): - with open(weight_file, 'rb') as w_f: - major, = struct.unpack('i', w_f.read(4)) - minor, = struct.unpack('i', w_f.read(4)) - revision, = struct.unpack('i', w_f.read(4)) - - if (major*10 + minor) >= 2 and major < 1000 and minor < 1000: - w_f.read(8) - else: - w_f.read(4) - - transpose = (major > 1000) or (minor > 1000) - - binary = w_f.read() - - self.offset = 0 - self.all_weights = np.frombuffer(binary, dtype='float32') - - def read_bytes(self, size): - self.offset = self.offset + size - return self.all_weights[self.offset-size:self.offset] - - def load_weights(self, model): - for i in range(106): - try: - conv_layer = model.get_layer('conv_' + str(i)) - print("loading weights of convolution #" + str(i)) - - if i not in [81, 93, 105]: - norm_layer = model.get_layer('bnorm_' + str(i)) - - size = np.prod(norm_layer.get_weights()[0].shape) - - beta = self.read_bytes(size) # bias - gamma = self.read_bytes(size) # scale - mean = self.read_bytes(size) # mean - var = self.read_bytes(size) # variance - - weights = norm_layer.set_weights([gamma, beta, mean, var]) - - if len(conv_layer.get_weights()) > 1: - bias = self.read_bytes(np.prod(conv_layer.get_weights()[1].shape)) - kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape)) - - kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape))) - kernel = kernel.transpose([2,3,1,0]) - conv_layer.set_weights([kernel, bias]) - else: - kernel = self.read_bytes(np.prod(conv_layer.get_weights()[0].shape)) - kernel = kernel.reshape(list(reversed(conv_layer.get_weights()[0].shape))) - kernel = kernel.transpose([2,3,1,0]) - conv_layer.set_weights([kernel]) - except ValueError: - print("no convolution #" + str(i)) - - def reset(self): - self.offset = 0 - -class BoundBox: - def __init__(self, xmin, ymin, xmax, ymax, objness = None, classes = None): - self.xmin = xmin - self.ymin = ymin - self.xmax = xmax - self.ymax = ymax - - self.objness = objness - self.classes = classes - - self.label = -1 - self.score = -1 - - def get_label(self): - if self.label == -1: - self.label = np.argmax(self.classes) - - return self.label - - def get_score(self): - if self.score == -1: - self.score = self.classes[self.get_label()] - - return self.score - def _conv_block(inp, convs, skip=True): x = inp count = 0 From 00498f8392283d8778102d6c43c306218b31fd88 Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 15:49:00 +0800 Subject: [PATCH 05/15] code reflection --- model/__init__.py | 0 model/__pycache__/__init__.cpython-36.pyc | Bin 0 -> 136 bytes model/__pycache__/yolo3.cpython-36.pyc | Bin 0 -> 3807 bytes model/yolo3.py | 160 ++++++++++ utils/__pycache__/tools.cpython-36.pyc | Bin 0 -> 9637 bytes utils/tools.py | 345 ++++++++++++++++++++++ yolo3_cam.py | 295 +----------------- yolo3_one_file_to_detect_them_all.py | 275 +---------------- 8 files changed, 521 insertions(+), 554 deletions(-) create mode 100644 model/__init__.py create mode 100644 model/__pycache__/__init__.cpython-36.pyc create mode 100644 model/__pycache__/yolo3.cpython-36.pyc create mode 100644 model/yolo3.py create mode 100644 utils/__pycache__/tools.cpython-36.pyc create mode 100644 utils/tools.py diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/model/__pycache__/__init__.cpython-36.pyc b/model/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19431153a4a7a01f316fdae748dd1a995385342c GIT binary patch literal 136 zcmXr!<>iu*V2x$~g2x~N1{i@12OutH0TL+;!3>&=ek&P@K*9*(m#KbeacWVqesWG~ zZfahMen3%vR%&udv3_=HQDU)fWqwY+v3_oTN@|XNe0*kJW=VX!UP0w84jZ6YX-=vg K$hcx4W&i+pUmq+0 literal 0 HcmV?d00001 diff --git a/model/__pycache__/yolo3.cpython-36.pyc b/model/__pycache__/yolo3.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4ef5f91ad961ca448c71505ea5c60958dc28293 GIT binary patch literal 3807 zcmaJ^UvJ#T5#L>scf8X{I!R|+k{wr#9Vh2Bk|nuL(i*Xl7%AGyvKz~A!z5s+Tlz$c z$2*ghEEzl?fc#JtFp$?4`2zk3?WZXE6zoG@`oy=k4}B;Cw7=Q=n`;^#_Q#!_ncdl$ z;qKh#^mO_5lJ&_qD~$b(jXehXTWHA_0Ldf|nZxlGp=cS7(J~#gWjR)>;1pU#r`WO` zo0FUomRe<}+?sNxG;M~J*0eK?{=6*6;&blINL!YkGiO$o3THv``>ei@!wR-jFQ~!??dWLZwz7U1bq-TC|Gt+t_IKKG%L{{NUK+F` zRoeEwgOiW^?FVYw@#FSAPs$)_LaFkgbKh%qLQU0$GQd$&jds-VQa?hgb7kFWOF!Ig zkOSx`qLtA;Kui7w$XM5qylWnRo$=f3v%hq$%uHFvPq`F}>_EgifO@Ksq;Vj9Rd^JHsUNGtfgeYHsH`KzLMk)yTE4R2-6L0(LhrO#M#_vx$v#YccQ5dg=C?2Z@z*U^oBx2YUW=%PDA@T7C@#UN3HAV9rxqD%|aX^+OG!x?{H$|D( z@U}&TRk+Oy_|5YfzQ}EM0c}OZZ$Pp^JE4b^DW!E9e}qoPJ{4WjH8PI3NlBcE0~TM; zEd9Pr-yPB#sdp+eB=&t4ErT*4v-KUGNj!;G7rhk0M`W~u2p?-Xh>#da2iF_x?7h4;OmDM> zj^vW3>@m;Sl>UCs_Y7@W}^Mc<4yF3 zzNO<~xagN4;5WVGW6=~D$KsrNr|`Gt7~q-yXi9p+97m^ilroPI8ba?_nJ2n` zLVfE|*!I$TS($rnM2B!gP3br!NK*IdII*uvLyi(p5zKwGrwHi&hO!PLB>n$*aSt=P zLHlr(yf_13e3>r`TM*VnOc86)Fg+!-Su`ES2)<33{J9h_c$cLJrNjX&O^L&k?v$?z zI3&3&Y1-7Z0d9|98uEKIC%S3E;~X<_+(84)slMbuv!ZF@_cW)qrZLpChilACNX}|W zifV6->Tr#@zT{wyc})|)x5k3j)N34Xr#RqG>5ADx-t=r8=+!bIImLQH|IGNf&MW>U zd-5%gbh>V6jQ%b1rI{_&r;ee$bZcaQDmOYwG!X`sc^T`?l~Wztgsu+LjAL zTVCm}J+S4XritI1%bu_Wm+hEe&JJz)(fFEjq`xvg-nT_OsXe4@BA*P_{qe-QuTI!? zX@Y-wg8v%PO-o*pufH(*(H~jyhL*1D)viG1s+JagX-nHO^6Q(L%ZGCrvV`v$JIB2E z`ZGE=s82f>Y5hd6F^b|_njZ02C-`d<{A)x{>|iw8x|Sa8U_(m}c2Ll^l!xBj)ZG8S z16^M9cQDeruGbiS6F=4Th<{^(|Mmp`oeBQCnjYEwGoqWtzF&(tCs9)2)%yg_TonlG7|XO`2B;<^TZ>ewM#RBwhPCU5c!RG{YM(2{i zCwN3aw`sml5CH1N&Q5%rI)#|Ls0$Hg#a`6(W7>aA`;Te=^%7kxDCFIs<$-$%V~Ph( zQI}AgH~O6$>rRnMBKMt*e&^lmj;%-5uia3VE}AB9oVk{F;JYMrwCQSzo0vU00x(`S zYP^cyS^Xv)^k(=4ep#6P@$aGfnpM7x-m-C#S9s+I$7r2pQRVRoRwnliAF5Wi&=Lx{ zum4~Z-2f6I~+r!b@ts5}7hwM_A2|%dWQk7{R3iT8DlxlMYIv6G7*Qu26JorK 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top + x = Conv2D(conv['filter'], + conv['kernel'], + strides=conv['stride'], + padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top + name='conv_' + str(conv['layer_idx']), + use_bias=False if conv['bnorm'] else True)(x) + if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) + if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) + + return add([skip_connection, x]) if skip else x + +def _interval_overlap(interval_a, interval_b): + x1, x2 = interval_a + x3, x4 = interval_b + + if x3 < x1: + if x4 < x1: + return 0 + else: + return min(x2,x4) - x1 + else: + if x2 < x3: + return 0 + else: + return min(x2,x4) - x3 + + +def _sigmoid(x): + return 1. / (1. + np.exp(-x)) + +def bbox_iou(box1, box2): + intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) + intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) + + intersect = intersect_w * intersect_h + + w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin + w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin + + union = w1*h1 + w2*h2 - intersect + + return float(intersect) / union + + +def make_yolov3_model(): + input_image = Input(shape=(None, None, 3)) + + # Layer 0 => 4 + x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0}, + {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1}, + {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2}, + {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}]) + + # Layer 5 => 8 + x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5}, + {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}]) + + # Layer 9 => 11 + x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}]) + + # Layer 12 => 15 + x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}]) + + # Layer 16 => 36 + for i in range(7): + x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}]) + + skip_36 = x + + # Layer 37 => 40 + x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}]) + + # Layer 41 => 61 + for i in range(7): + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}]) + + skip_61 = x + + # Layer 62 => 65 + x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}]) + + # Layer 66 => 74 + for i in range(3): + x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}]) + + # Layer 75 => 79 + x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False) + + # Layer 80 => 82 + yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False) + + # Layer 83 => 86 + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_61]) + + # Layer 87 => 91 + x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False) + + # Layer 92 => 94 + yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False) + + # Layer 95 => 98 + x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}], skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_36]) + + # Layer 99 => 106 + yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104}, + {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False) + + model = Model(input_image, [yolo_82, yolo_94, yolo_106]) + return model + +class YOLO3(): + ''' + To be continued + ''' + pass \ No newline at end of file diff --git a/utils/__pycache__/tools.cpython-36.pyc b/utils/__pycache__/tools.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19de412478dd3283bbe49454a893f70c5483e6a6 GIT binary patch literal 9637 zcmcgyO>88`b?)lF>G|PsNX~M(+|{bJTFaucKgvpx?Zv|SU3(o_lpw9M(pZ~GbE-Mh z!; z!y%W79Uw>#y1M#R)vH&p-e0|*7Zw&Ox7Mxizy0f)_RrefWuyKI?)cA9aE-Hp)>2Q_ z(&f9+GPur-fjP8V*3fR*jM~jXY3Q^Z)Gcm5)LLa;;trk_eu>xk!b8@o@;YAxq{bV3 z3C{(-%vbQN^Cmxo=OSO_XYp+CHGU4yCBDwjrtFLLvP3Dfr#_c+wa{LA`-bd@{(R|1u?)_ z*#c&fOHu_bT@VQs+%Xjz(^(zA8k6v!6G+p3`tr~J_HXF=$;*d-3g&faAYD)4rZpM3 z`#s`Bh1QZSOd+hr(g56E7-61vH6Vzo1jmKz+J?2y>W3;@E&AXwvf%SNJ30$25_f0Nr$Pry_*yZYMo- z??nR+q>6$g$T;@7i;HQ5+;-Bsk=Te%GV=5zWz7 zznDT1{bf_|AO^#_;~?(2TYj?VVjG_DtNH;lro>UGAxa2Wq;m_e)<2BA;XgC)XE z_M)RD#I7g6rtxSHbRZQ++k?Gp_XB<&i>JAFx%c^v+V^u>HV_FW;kN+f!`*7RNzq* zO8|(YRz+&;?m=kX2SGgc2Ej+z$ykI(40+Y%0k&i4jYgy>oq-p} zZs-kt8srXNeaDSE5tdpg%`_Gu&rA+~gG^y}6j z>Ue|ro7avy&Q*ce5?~8+;+XPG1?Rke6V3DUUHG5oJGR{a z7iLNF1?|ZRehDJ{Io$tyMu{_oZWTqr9&wiHOso-{!ZaZRq7yR+l9Xc{m~A-ql;Tp$ zSjzg^L9yu)nc{rXmQpv}5CbvkX)C$`OvQ>$pNh6qPE}D;De0O{WDUG6e=uFvI{KbA z$qQdH2`ya;@H5j%E~-heCR&${+rW*;t82|qY~|(zrYkr1WjPso6RLX?aSoV%895a4 z7Hi%-H09JN=n|v^;GgMDf*2b_J8{wU5XQ(WF?r1zs0#!|&vW#R29_O&W-&s?z%D<*^4s!EWQu((x`u(o8{d2CBwox|2zh7lL= zHf;>)1TlCR=Zo90UVs#SIL_;d7|*}k$}YDS$$-sctIQGJRCUlc*CS^QlNpg40CZ8cz7Gb+Gg z7ML*BaM(q_>bm$gX6=v>rmKdVROfZvaRYjSldO5bQiPkR8K|-I+KwikOU=wmS!y;l zYRl}f0f?RIsR0Q3WY*-z2}^W3U(%87{*SU!YNpm2yx@7@l=v**LrvE*CoAI+W*4o7 zY_-vf8d@vGk*#zN$FoA{9?i@g$0kT5d@Ll9!}iY4@g; zJ@F0dbQ1+B++KT6J$h1?atGqJy^Vl9FHKv~|c*(naz~cAR`u{5cA2KiJI~&bt?=&Ka0ZXIniq zDQz(MGiyD^AkJWRmC@4MCDUdzndRZ=8lKY@|gPKhe$s z_l_=--Mf=-0jJ4LoPE?79fI^c#0;W>F{Pv!rG(%Ber3lJpYTE`2p7)FpWuXh7x#0; z23nhsdtefI3JxTLq+4t#k44l}Pk7H9S|DxQEu6ji9iHk~Gpr{0ufC z)%ws4&OXNB`a?alGNekN?bu98v82e{ZNr9Ssvi#b++2gkHEo|m|E8g@S^Ba(HI;2n5 z5}W8HtMCTi@7qXYR8o6g%c}HTMRI8J3-l>T(fwJr0I{ngRkOhDZ4%)+k~aDdd&I=* zBW?FTW~~~xK^=O0Xl9F9BU{RrDfM$!OKsT6Mc` zAecsLfZF`yJy=_uQ5H52NtO@EAV`rtB%g6jxb#i|)!~2P65mH{Ls#I(o)tlnc3z?* zIAR}}z7!Z2~t8MIhG+H;fv#9j^E#SVOqN)P$g-mYL*=$+@Kl zAgEUK24c67h!D=ju%`&c{~qlz9q(WmjWz6=jlDCCD@K#ugzpp)T4skU(g4yAxpRaM z{D#`m;l2nMXt6XK4jwyrsF{zZFNcvB%KXH?pjG+}B=|?Dcc>FxB)K!gTO-2@g8-jN z5>lcNO36i5k^5btV2s1?!1PupmlKmR#_*qOl#Rw7!jMvr1A~kh{N#&JbPc;zMgIkk zymeZ^-q+#Rrz3xmk+H*!a_j=ZQ^8>)4UY!PO?F}ahNpamIXIpf!1y5PmrC6W{v zH{@qkjT{CHR|C0pm?Yp>h5;s;EM$ipyMJ*{g8^$m_8bMpM~gDvAx(@-7c%IeI|SWz z@Yz;=T?&+xF)2)kq|h76@F*869V?|Ol%J(t%xUi_+tZH1_Li4)dQvTAL?nIR#)|k2 z>Ss&IiRYy3cL@g)=*%#OpB4R~?H_Z4{aOAy*+c7a_m8b(Ej;8Oot zphswkM^bQquMjHo@D~De=Hd+epk$ zM@!fN^)-(5m9ub*Lb>^Q4|GHf)bluINy7=u5_Sq^N#QZf5+X61LG#e!kUqUsR z?0o%KOvsC^ERS!31lw;Q#sD7vrA6m-E^PbyyCvN z;f7phG$;=%PdBo83(aqSdGmbow7@HeO%Kxw`J2a(7j3irw-kMu{@xw?VqYZbs0iUD^hQj_1LPXKq1XpytK&r&9Q z7RkIFr_UXysa+Q`Ag`kGi~)T@8Ue{?D+nOl9wB15w3X@*0+ICq$oBF8Ei`>ZW(_2u{`!~qtnYoF;Szj zCY+fdSjVJyToh(ia7$JD2`ZC+N-4tmyUgSeX1lES^l;brEye+(x#+#ur%|yfJZ9GyBdq z6blC^vtQZP@dp*>z(UMo^{eP%@p6H)Mqm2j5Q;Gz38`%zy%sq7_3IqD4Ma(cymnB+ zTsGz+t~NGvW%xeQZ@PyZ%l(3Vr8s%vMaDf}F!zxDlhOUp&;IQn-r9Pn2&m1OMqEjsS~F-ku@odnj^iBw!aIZ;22+0+bue3j#p=`ES)q8wLVLLhV3 zpCmF}pEtkt_U7I8o!jrecjxv8?f1U();r(5{k{z7k3OB zmGFL>3d*g?NKyrs6m-hiUdCe5fk-8k@-2y+TQ5auDr4i^LMnee@V`kL7n57U(Govo feRaoh%J^4L@v97+C8y~;>zo1Hb 0: + pred_boxes = np.array([[box.xmin, box.ymin, box.xmax, box.ymax, box.get_score()] for box in pred_boxes]) + else: + pred_boxes = np.array([[]]) + + # sort the boxes and the labels according to scores + score_sort = np.argsort(-score) + pred_labels = pred_labels[score_sort] + pred_boxes = pred_boxes[score_sort] + + # copy detections to all_detections + for label in range(generator.num_classes()): + all_detections[i][label] = pred_boxes[pred_labels == label, :] + + annotations = generator.load_annotation(i) + + # copy detections to all_annotations + for label in range(generator.num_classes()): + all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy() + + # compute mAP by comparing all detections and all annotations + average_precisions = {} + + for label in range(generator.num_classes()): + false_positives = np.zeros((0,)) + true_positives = np.zeros((0,)) + scores = np.zeros((0,)) + num_annotations = 0.0 + + for i in range(generator.size()): + detections = all_detections[i][label] + annotations = all_annotations[i][label] + num_annotations += annotations.shape[0] + detected_annotations = [] + + for d in detections: + scores = np.append(scores, d[4]) + + if annotations.shape[0] == 0: + false_positives = np.append(false_positives, 1) + true_positives = np.append(true_positives, 0) + continue + + overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations) + assigned_annotation = np.argmax(overlaps, axis=1) + max_overlap = overlaps[0, assigned_annotation] + + if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations: + false_positives = np.append(false_positives, 0) + true_positives = np.append(true_positives, 1) + detected_annotations.append(assigned_annotation) + else: + false_positives = np.append(false_positives, 1) + true_positives = np.append(true_positives, 0) + + # no annotations -> AP for this class is 0 (is this correct?) + if num_annotations == 0: + average_precisions[label] = 0 + continue + + # sort by score + indices = np.argsort(-scores) + false_positives = false_positives[indices] + true_positives = true_positives[indices] + + # compute false positives and true positives + false_positives = np.cumsum(false_positives) + true_positives = np.cumsum(true_positives) + + # compute recall and precision + recall = true_positives / num_annotations + precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps) + + # compute average precision + average_precision = compute_ap(recall, precision) + average_precisions[label] = average_precision + + return average_precisions + +def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): + if (float(net_w)/image_w) < (float(net_h)/image_h): + new_w = net_w + new_h = (image_h*net_w)/image_w + else: + new_h = net_w + new_w = (image_w*net_h)/image_h + + for i in range(len(boxes)): + x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w + y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h + + boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) + boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) + boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) + boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) + +def do_nms(boxes, nms_thresh): + if len(boxes) > 0: + nb_class = len(boxes[0].classes) + else: + return + + for c in range(nb_class): + sorted_indices = np.argsort([-box.classes[c] for box in boxes]) + + for i in range(len(sorted_indices)): + index_i = sorted_indices[i] + + if boxes[index_i].classes[c] == 0: continue + + for j in range(i+1, len(sorted_indices)): + index_j = sorted_indices[j] + + if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: + boxes[index_j].classes[c] = 0 + +def decode_netout(netout, anchors, obj_thresh, net_h, net_w): + grid_h, grid_w = netout.shape[:2] + nb_box = 3 + netout = netout.reshape((grid_h, grid_w, nb_box, -1)) + nb_class = netout.shape[-1] - 5 + + boxes = [] + + netout[..., :2] = _sigmoid(netout[..., :2]) + netout[..., 4] = _sigmoid(netout[..., 4]) + netout[..., 5:] = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:]) + netout[..., 5:] *= netout[..., 5:] > obj_thresh + + for i in range(grid_h*grid_w): + row = i // grid_w + col = i % grid_w + + for b in range(nb_box): + # 4th element is objectness score + objectness = netout[row, col, b, 4] + + if(objectness <= obj_thresh): continue + + # first 4 elements are x, y, w, and h + x, y, w, h = netout[row,col,b,:4] + + x = (col + x) / grid_w # center position, unit: image width + y = (row + y) / grid_h # center position, unit: image height + w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width + h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height + + # last elements are class probabilities + classes = netout[row,col,b,5:] + + box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes) + + boxes.append(box) + + return boxes + +def preprocess_input(image, net_h, net_w): + new_h, new_w, _ = image.shape + + # determine the new size of the image + if (float(net_w)/new_w) < (float(net_h)/new_h): + new_h = (new_h * net_w)//new_w + new_w = net_w + else: + new_w = (new_w * net_h)//new_h + new_h = net_h + + # resize the image to the new size + resized = cv2.resize(image[:,:,::-1]/255., (new_w, new_h)) + + # embed the image into the standard letter box + new_image = np.ones((net_h, net_w, 3)) * 0.5 + new_image[(net_h-new_h)//2:(net_h+new_h)//2, (net_w-new_w)//2:(net_w+new_w)//2, :] = resized + new_image = np.expand_dims(new_image, 0) + + return new_image + +def normalize(image): + return image/255. + +def get_yolo_boxes(model, images, net_h, net_w, anchors, obj_thresh, nms_thresh): + image_h, image_w, _ = images[0].shape + nb_images = len(images) + batch_input = np.zeros((nb_images, net_h, net_w, 3)) + + # preprocess the input + for i in range(nb_images): + batch_input[i] = preprocess_input(images[i], net_h, net_w) + + # run the prediction + batch_output = model.predict_on_batch(batch_input) + batch_boxes = [None]*nb_images + + for i in range(nb_images): + yolos = [batch_output[0][i], batch_output[1][i], batch_output[2][i]] + boxes = [] + + # decode the output of the network + for j in range(len(yolos)): + yolo_anchors = anchors[(2-j)*6:(3-j)*6] # config['model']['anchors'] + boxes += decode_netout(yolos[j], yolo_anchors, obj_thresh, net_h, net_w) + + # correct the sizes of the bounding boxes + correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) + + # suppress non-maximal boxes + do_nms(boxes, nms_thresh) + + batch_boxes[i] = boxes + + return batch_boxes + +def compute_overlap(a, b): + """ + Code originally from https://github.com/rbgirshick/py-faster-rcnn. + Parameters + ---------- + a: (N, 4) ndarray of float + b: (K, 4) ndarray of float + Returns + ------- + overlaps: (N, K) ndarray of overlap between boxes and query_boxes + """ + area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1]) + + iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0]) + ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1]) + + iw = np.maximum(iw, 0) + ih = np.maximum(ih, 0) + + ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih + + ua = np.maximum(ua, np.finfo(float).eps) + + intersection = iw * ih + + return intersection / ua + +def compute_ap(recall, precision): + """ Compute the average precision, given the recall and precision curves. + Code originally from https://github.com/rbgirshick/py-faster-rcnn. + + # Arguments + recall: The recall curve (list). + precision: The precision curve (list). + # Returns + The average precision as computed in py-faster-rcnn. + """ + # correct AP calculation + # first append sentinel values at the end + mrec = np.concatenate(([0.], recall, [1.])) + mpre = np.concatenate(([0.], precision, [0.])) + + # compute the precision envelope + for i in range(mpre.size - 1, 0, -1): + mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) + + # to calculate area under PR curve, look for points + # where X axis (recall) changes value + i = np.where(mrec[1:] != mrec[:-1])[0] + + # and sum (\Delta recall) * prec + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) + return ap + +def _softmax(x, axis=-1): + x = x - np.amax(x, axis, keepdims=True) + e_x = np.exp(x) + + return e_x / e_x.sum(axis, keepdims=True) + +def draw_boxes(image, boxes, labels, obj_thresh): + for box in boxes: + label_str = '' + label = -1 + + for i in range(len(labels)): + if box.classes[i] > obj_thresh: + label_str += labels[i] + label = i + print(labels[i] + ': ' + str(box.classes[i]*100) + '%') + + if label >= 0: + cv2.rectangle(image, (box.xmin,box.ymin), (box.xmax,box.ymax), (0,255,0), 3) + cv2.putText(image, + label_str + ' ' + str(box.get_score()), + (box.xmin, box.ymin - 13), + cv2.FONT_HERSHEY_SIMPLEX, + 1e-3 * image.shape[0], + (0,255,0), 2) + + return image \ No newline at end of file diff --git a/yolo3_cam.py b/yolo3_cam.py index 1e8960421..1f816e9ab 100644 --- a/yolo3_cam.py +++ b/yolo3_cam.py @@ -1,296 +1,19 @@ import argparse import os -import numpy as np -from keras.layers import Conv2D, Input, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D -from keras.layers.merge import add, concatenate -from keras.models import Model + import struct import cv2 +import numpy as np from utils.weightreader import WeightReader from utils.bbox import BoundBox +from utils.tools import preprocess_input, decode_netout +from utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"]="0" -argparser = argparse.ArgumentParser( - description='test yolov3 network with coco weights') - -argparser.add_argument( - '-w', - '--weights', - help='path to weights file') - -def _conv_block(inp, convs, skip=True): - x = inp - count = 0 - - for conv in convs: - if count == (len(convs) - 2) and skip: - skip_connection = x - count += 1 - - if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top - x = Conv2D(conv['filter'], - conv['kernel'], - strides=conv['stride'], - padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top - name='conv_' + str(conv['layer_idx']), - use_bias=False if conv['bnorm'] else True)(x) - if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) - if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) - - return add([skip_connection, x]) if skip else x - -def _interval_overlap(interval_a, interval_b): - x1, x2 = interval_a - x3, x4 = interval_b - - if x3 < x1: - if x4 < x1: - return 0 - else: - return min(x2,x4) - x1 - else: - if x2 < x3: - return 0 - else: - return min(x2,x4) - x3 - -def _sigmoid(x): - return 1. / (1. + np.exp(-x)) - -def bbox_iou(box1, box2): - intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) - intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) - - intersect = intersect_w * intersect_h - - w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin - w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin - - union = w1*h1 + w2*h2 - intersect - - return float(intersect) / union - -def make_yolov3_model(): - input_image = Input(shape=(None, None, 3)) - - # Layer 0 => 4 - x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0}, - {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1}, - {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2}, - {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}]) - - # Layer 5 => 8 - x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5}, - {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6}, - {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}]) - - # Layer 9 => 11 - x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9}, - {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}]) - - # Layer 12 => 15 - x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}]) - - # Layer 16 => 36 - for i in range(7): - x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}]) - - skip_36 = x - - # Layer 37 => 40 - x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}]) - - # Layer 41 => 61 - for i in range(7): - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}]) - - skip_61 = x - - # Layer 62 => 65 - x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}]) - - # Layer 66 => 74 - for i in range(3): - x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}]) - - # Layer 75 => 79 - x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False) - - # Layer 80 => 82 - yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False) - - # Layer 83 => 86 - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False) - x = UpSampling2D(2)(x) - x = concatenate([x, skip_61]) - - # Layer 87 => 91 - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False) - - # Layer 92 => 94 - yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False) - - # Layer 95 => 98 - x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}], skip=False) - x = UpSampling2D(2)(x) - x = concatenate([x, skip_36]) - - # Layer 99 => 106 - yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False) - - model = Model(input_image, [yolo_82, yolo_94, yolo_106]) - return model - -def preprocess_input(image, net_h, net_w): - new_h, new_w, _ = image.shape - - # determine the new size of the image - if (float(net_w)/new_w) < (float(net_h)/new_h): - new_h = (new_h * net_w)/new_w - new_w = net_w - else: - new_w = (new_w * net_h)/new_h - new_h = net_h - - # resize the image to the new size - resized = cv2.resize(image[:,:,::-1]/255., (int(new_w), int(new_h))) - - # embed the image into the standard letter box - new_image = np.ones((net_h, net_w, 3)) * 0.5 - new_image[int((net_h-new_h)//2):int((net_h+new_h)//2), int((net_w-new_w)//2):int((net_w+new_w)//2), :] = resized - new_image = np.expand_dims(new_image, 0) - - return new_image - -def decode_netout(netout, anchors, obj_thresh, nms_thresh, net_h, net_w): - grid_h, grid_w = netout.shape[:2] - nb_box = 3 - netout = netout.reshape((grid_h, grid_w, nb_box, -1)) - nb_class = netout.shape[-1] - 5 - - boxes = [] - - netout[..., :2] = _sigmoid(netout[..., :2]) - netout[..., 4:] = _sigmoid(netout[..., 4:]) - netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:] - netout[..., 5:] *= netout[..., 5:] > obj_thresh - - for i in range(grid_h*grid_w): - row = i / grid_w - col = i % grid_w - - for b in range(nb_box): - # 4th element is objectness score - objectness = netout[int(row)][int(col)][b][4] - #objectness = netout[..., :4] - - if(objectness.all() <= obj_thresh): continue - - # first 4 elements are x, y, w, and h - x, y, w, h = netout[int(row)][int(col)][b][:4] - - x = (col + x) / grid_w # center position, unit: image width - y = (row + y) / grid_h # center position, unit: image height - w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width - h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height - - # last elements are class probabilities - classes = netout[int(row)][col][b][5:] - - box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes) - #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes) - - boxes.append(box) - - return boxes - -def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): - if (float(net_w)/image_w) < (float(net_h)/image_h): - new_w = net_w - new_h = (image_h*net_w)/image_w - else: - new_h = net_w - new_w = (image_w*net_h)/image_h - - for i in range(len(boxes)): - x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w - y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h - - boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) - boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) - boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) - boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) - -def do_nms(boxes, nms_thresh): - if len(boxes) > 0: - nb_class = len(boxes[0].classes) - else: - return - - for c in range(nb_class): - sorted_indices = np.argsort([-box.classes[c] for box in boxes]) - - for i in range(len(sorted_indices)): - index_i = sorted_indices[i] - - if boxes[index_i].classes[c] == 0: continue - - for j in range(i+1, len(sorted_indices)): - index_j = sorted_indices[j] - - if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: - boxes[index_j].classes[c] = 0 - -def draw_boxes(image, boxes, labels, obj_thresh): - for box in boxes: - label_str = '' - label = -1 - - for i in range(len(labels)): - if box.classes[i] > obj_thresh: - label_str += labels[i] - label = i - print(labels[i] + ': ' + str(box.classes[i]*100) + '%') - - if label >= 0: - cv2.rectangle(image, (box.xmin,box.ymin), (box.xmax,box.ymax), (0,255,0), 3) - cv2.putText(image, - label_str + ' ' + str(box.get_score()), - (box.xmin, box.ymin - 13), - cv2.FONT_HERSHEY_SIMPLEX, - 1e-3 * image.shape[0], - (0,255,0), 2) - - return image - def _main_(args): weights_path = args.weights image_path = args.image @@ -328,7 +51,7 @@ def _main_(args): for i in range(len(yolos)): # decode the output of the network - boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, nms_thresh, net_h, net_w) + boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) # correct the sizes of the bounding boxes correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) @@ -343,5 +66,11 @@ def _main_(args): cv2.imwrite(image_path[:-4] + '_detected' + image_path[-4:], (image).astype('uint8')) if __name__ == '__main__': + argparser = argparse.ArgumentParser( + description='test yolov3 network with coco weights') + argparser.add_argument( + '-w', + '--weights', + help='path to weights file') args = argparser.parse_args() _main_(args) diff --git a/yolo3_one_file_to_detect_them_all.py b/yolo3_one_file_to_detect_them_all.py index e5b77a1a0..4d59fd7cb 100644 --- a/yolo3_one_file_to_detect_them_all.py +++ b/yolo3_one_file_to_detect_them_all.py @@ -8,6 +8,9 @@ import cv2 from utils.weightreader import WeightReader from utils.bbox import BoundBox +from utils.tools import preprocess_input, decode_netout +from utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" @@ -26,276 +29,6 @@ '--image', help='path to image file') -def _conv_block(inp, convs, skip=True): - x = inp - count = 0 - - for conv in convs: - if count == (len(convs) - 2) and skip: - skip_connection = x - count += 1 - - if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top - x = Conv2D(conv['filter'], - conv['kernel'], - strides=conv['stride'], - padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top - name='conv_' + str(conv['layer_idx']), - use_bias=False if conv['bnorm'] else True)(x) - if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) - if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) - - return add([skip_connection, x]) if skip else x - -def _interval_overlap(interval_a, interval_b): - x1, x2 = interval_a - x3, x4 = interval_b - - if x3 < x1: - if x4 < x1: - return 0 - else: - return min(x2,x4) - x1 - else: - if x2 < x3: - return 0 - else: - return min(x2,x4) - x3 - -def _sigmoid(x): - return 1. / (1. + np.exp(-x)) - -def bbox_iou(box1, box2): - intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax]) - intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax]) - - intersect = intersect_w * intersect_h - - w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin - w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin - - union = w1*h1 + w2*h2 - intersect - - return float(intersect) / union - -def make_yolov3_model(): - input_image = Input(shape=(None, None, 3)) - - # Layer 0 => 4 - x = _conv_block(input_image, [{'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0}, - {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1}, - {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2}, - {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}]) - - # Layer 5 => 8 - x = _conv_block(x, [{'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5}, - {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6}, - {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}]) - - # Layer 9 => 11 - x = _conv_block(x, [{'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9}, - {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}]) - - # Layer 12 => 15 - x = _conv_block(x, [{'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}]) - - # Layer 16 => 36 - for i in range(7): - x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}]) - - skip_36 = x - - # Layer 37 => 40 - x = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}]) - - # Layer 41 => 61 - for i in range(7): - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}]) - - skip_61 = x - - # Layer 62 => 65 - x = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}]) - - # Layer 66 => 74 - for i in range(3): - x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}]) - - # Layer 75 => 79 - x = _conv_block(x, [{'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77}, - {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78}, - {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79}], skip=False) - - # Layer 80 => 82 - yolo_82 = _conv_block(x, [{'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81}], skip=False) - - # Layer 83 => 86 - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84}], skip=False) - x = UpSampling2D(2)(x) - x = concatenate([x, skip_61]) - - # Layer 87 => 91 - x = _conv_block(x, [{'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89}, - {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90}, - {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91}], skip=False) - - # Layer 92 => 94 - yolo_94 = _conv_block(x, [{'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93}], skip=False) - - # Layer 95 => 98 - x = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96}], skip=False) - x = UpSampling2D(2)(x) - x = concatenate([x, skip_36]) - - # Layer 99 => 106 - yolo_106 = _conv_block(x, [{'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102}, - {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103}, - {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104}, - {'filter': 255, 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105}], skip=False) - - model = Model(input_image, [yolo_82, yolo_94, yolo_106]) - return model - -def preprocess_input(image, net_h, net_w): - new_h, new_w, _ = image.shape - - # determine the new size of the image - if (float(net_w)/new_w) < (float(net_h)/new_h): - new_h = (new_h * net_w)/new_w - new_w = net_w - else: - new_w = (new_w * net_h)/new_h - new_h = net_h - - # resize the image to the new size - resized = cv2.resize(image[:,:,::-1]/255., (int(new_w), int(new_h))) - - # embed the image into the standard letter box - new_image = np.ones((net_h, net_w, 3)) * 0.5 - new_image[int((net_h-new_h)//2):int((net_h+new_h)//2), int((net_w-new_w)//2):int((net_w+new_w)//2), :] = resized - new_image = np.expand_dims(new_image, 0) - - return new_image - -def decode_netout(netout, anchors, obj_thresh, nms_thresh, net_h, net_w): - grid_h, grid_w = netout.shape[:2] - nb_box = 3 - netout = netout.reshape((grid_h, grid_w, nb_box, -1)) - nb_class = netout.shape[-1] - 5 - - boxes = [] - - netout[..., :2] = _sigmoid(netout[..., :2]) - netout[..., 4:] = _sigmoid(netout[..., 4:]) - netout[..., 5:] = netout[..., 4][..., np.newaxis] * netout[..., 5:] - netout[..., 5:] *= netout[..., 5:] > obj_thresh - - for i in range(grid_h*grid_w): - row = i / grid_w - col = i % grid_w - - for b in range(nb_box): - # 4th element is objectness score - objectness = netout[int(row)][int(col)][b][4] - #objectness = netout[..., :4] - - if(objectness.all() <= obj_thresh): continue - - # first 4 elements are x, y, w, and h - x, y, w, h = netout[int(row)][int(col)][b][:4] - - x = (col + x) / grid_w # center position, unit: image width - y = (row + y) / grid_h # center position, unit: image height - w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width - h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height - - # last elements are class probabilities - classes = netout[int(row)][col][b][5:] - - box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes) - #box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, None, classes) - - boxes.append(box) - - return boxes - -def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w): - if (float(net_w)/image_w) < (float(net_h)/image_h): - new_w = net_w - new_h = (image_h*net_w)/image_w - else: - new_h = net_w - new_w = (image_w*net_h)/image_h - - for i in range(len(boxes)): - x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w - y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h - - boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w) - boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w) - boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h) - boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h) - -def do_nms(boxes, nms_thresh): - if len(boxes) > 0: - nb_class = len(boxes[0].classes) - else: - return - - for c in range(nb_class): - sorted_indices = np.argsort([-box.classes[c] for box in boxes]) - - for i in range(len(sorted_indices)): - index_i = sorted_indices[i] - - if boxes[index_i].classes[c] == 0: continue - - for j in range(i+1, len(sorted_indices)): - index_j = sorted_indices[j] - - if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh: - boxes[index_j].classes[c] = 0 - -def draw_boxes(image, boxes, labels, obj_thresh): - for box in boxes: - label_str = '' - label = -1 - - for i in range(len(labels)): - if box.classes[i] > obj_thresh: - label_str += labels[i] - label = i - print(labels[i] + ': ' + str(box.classes[i]*100) + '%') - - if label >= 0: - cv2.rectangle(image, (box.xmin,box.ymin), (box.xmax,box.ymax), (0,255,0), 3) - cv2.putText(image, - label_str + ' ' + str(box.get_score()), - (box.xmin, box.ymin - 13), - cv2.FONT_HERSHEY_SIMPLEX, - 1e-3 * image.shape[0], - (0,255,0), 2) - - return image - def _main_(args): weights_path = args.weights image_path = args.image @@ -333,7 +66,7 @@ def _main_(args): for i in range(len(yolos)): # decode the output of the network - boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, nms_thresh, net_h, net_w) + boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) # correct the sizes of the bounding boxes correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) From 9007ed5cdca0d94abc5af6b8450ae792aa7f8cce Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 16:10:29 +0800 Subject: [PATCH 06/15] fin cam demo --- utils/__pycache__/tools.cpython-36.pyc | Bin 9637 -> 9637 bytes utils/tools.py | 2 +- yolo3_cam.py | 90 ++++++++++++++----------- 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/utils/__pycache__/tools.cpython-36.pyc b/utils/__pycache__/tools.cpython-36.pyc index 19de412478dd3283bbe49454a893f70c5483e6a6..d5cc22e896b4ca268a7ee20a46fd99326d418a94 100644 GIT binary patch delta 18 ZcmZ4Lz0{l2n3tE!M}alkb0g;*RRAxV1knHh delta 18 ZcmZ4Lz0{l2n3tE!Mv67se Date: Wed, 23 May 2018 16:16:22 +0800 Subject: [PATCH 07/15] update gitignore --- .gitignore | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7d7b138f9..506c445eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,61 @@ +# Data files and directories common in repo root +datasets/ +logs/ +*.h5 +*.weights +results/ +temp/ +test/ *.jpg *.jpeg -*.weights -*.h5 + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# VS Studio Code +.vscode + +# PyCharm +.idea/ + +# Dropbox +.dropbox.attr + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ From 2fe9403e45a979b3c270c9924f4b324baa2d181b Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 23 May 2018 16:17:14 +0800 Subject: [PATCH 08/15] all pyc files removed --- model/__pycache__/__init__.cpython-36.pyc | Bin 136 -> 0 bytes model/__pycache__/yolo3.cpython-36.pyc | Bin 3807 -> 0 bytes utils/__pycache__/__init__.cpython-36.pyc | Bin 136 -> 0 bytes utils/__pycache__/bbox.cpython-36.pyc | Bin 2619 -> 0 bytes utils/__pycache__/colors.cpython-36.pyc | Bin 1685 -> 0 bytes utils/__pycache__/tools.cpython-36.pyc | Bin 9637 -> 0 bytes utils/__pycache__/weightreader.cpython-36.pyc | Bin 2368 -> 0 bytes 7 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 model/__pycache__/__init__.cpython-36.pyc delete mode 100644 model/__pycache__/yolo3.cpython-36.pyc delete mode 100644 utils/__pycache__/__init__.cpython-36.pyc delete mode 100644 utils/__pycache__/bbox.cpython-36.pyc delete mode 100644 utils/__pycache__/colors.cpython-36.pyc delete mode 100644 utils/__pycache__/tools.cpython-36.pyc delete mode 100644 utils/__pycache__/weightreader.cpython-36.pyc diff --git a/model/__pycache__/__init__.cpython-36.pyc b/model/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index 19431153a4a7a01f316fdae748dd1a995385342c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 136 zcmXr!<>iu*V2x$~g2x~N1{i@12OutH0TL+;!3>&=ek&P@K*9*(m#KbeacWVqesWG~ zZfahMen3%vR%&udv3_=HQDU)fWqwY+v3_oTN@|XNe0*kJW=VX!UP0w84jZ6YX-=vg K$hcx4W&i+pUmq+0 diff --git a/model/__pycache__/yolo3.cpython-36.pyc b/model/__pycache__/yolo3.cpython-36.pyc deleted file mode 100644 index a4ef5f91ad961ca448c71505ea5c60958dc28293..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3807 zcmaJ^UvJ#T5#L>scf8X{I!R|+k{wr#9Vh2Bk|nuL(i*Xl7%AGyvKz~A!z5s+Tlz$c z$2*ghEEzl?fc#JtFp$?4`2zk3?WZXE6zoG@`oy=k4}B;Cw7=Q=n`;^#_Q#!_ncdl$ z;qKh#^mO_5lJ&_qD~$b(jXehXTWHA_0Ldf|nZxlGp=cS7(J~#gWjR)>;1pU#r`WO` zo0FUomRe<}+?sNxG;M~J*0eK?{=6*6;&blINL!YkGiO$o3THv``>ei@!wR-jFQ~!??dWLZwz7U1bq-TC|Gt+t_IKKG%L{{NUK+F` zRoeEwgOiW^?FVYw@#FSAPs$)_LaFkgbKh%qLQU0$GQd$&jds-VQa?hgb7kFWOF!Ig zkOSx`qLtA;Kui7w$XM5qylWnRo$=f3v%hq$%uHFvPq`F}>_EgifO@Ksq;Vj9Rd^JHsUNGtfgeYHsH`KzLMk)yTE4R2-6L0(LhrO#M#_vx$v#YccQ5dg=C?2Z@z*U^oBx2YUW=%PDA@T7C@#UN3HAV9rxqD%|aX^+OG!x?{H$|D( z@U}&TRk+Oy_|5YfzQ}EM0c}OZZ$Pp^JE4b^DW!E9e}qoPJ{4WjH8PI3NlBcE0~TM; zEd9Pr-yPB#sdp+eB=&t4ErT*4v-KUGNj!;G7rhk0M`W~u2p?-Xh>#da2iF_x?7h4;OmDM> zj^vW3>@m;Sl>UCs_Y7@W}^Mc<4yF3 zzNO<~xagN4;5WVGW6=~D$KsrNr|`Gt7~q-yXi9p+97m^ilroPI8ba?_nJ2n` zLVfE|*!I$TS($rnM2B!gP3br!NK*IdII*uvLyi(p5zKwGrwHi&hO!PLB>n$*aSt=P zLHlr(yf_13e3>r`TM*VnOc86)Fg+!-Su`ES2)<33{J9h_c$cLJrNjX&O^L&k?v$?z zI3&3&Y1-7Z0d9|98uEKIC%S3E;~X<_+(84)slMbuv!ZF@_cW)qrZLpChilACNX}|W zifV6->Tr#@zT{wyc})|)x5k3j)N34Xr#RqG>5ADx-t=r8=+!bIImLQH|IGNf&MW>U zd-5%gbh>V6jQ%b1rI{_&r;ee$bZcaQDmOYwG!X`sc^T`?l~Wztgsu+LjAL zTVCm}J+S4XritI1%bu_Wm+hEe&JJz)(fFEjq`xvg-nT_OsXe4@BA*P_{qe-QuTI!? zX@Y-wg8v%PO-o*pufH(*(H~jyhL*1D)viG1s+JagX-nHO^6Q(L%ZGCrvV`v$JIB2E z`ZGE=s82f>Y5hd6F^b|_njZ02C-`d<{A)x{>|iw8x|Sa8U_(m}c2Ll^l!xBj)ZG8S z16^M9cQDeruGbiS6F=4Th<{^(|Mmp`oeBQCnjYEwGoqWtzF&(tCs9)2)%yg_TonlG7|XO`2B;<^TZ>ewM#RBwhPCU5c!RG{YM(2{i zCwN3aw`sml5CH1N&Q5%rI)#|Ls0$Hg#a`6(W7>aA`;Te=^%7kxDCFIs<$-$%V~Ph( zQI}AgH~O6$>rRnMBKMt*e&^lmj;%-5uia3VE}AB9oVk{F;JYMrwCQSzo0vU00x(`S zYP^cyS^Xv)^k(=4ep#6P@$aGfnpM7x-m-C#S9s+I$7r2pQRVRoRwnliAF5Wi&=Lx{ zum4~Z-2f6I~+r!b@ts5}7hwM_A2|%dWQk7{R3iT8DlxlMYIv6G7*Qu26JorKiupeJzRs2p)q77+?f49Dul(1xTbY1T$zd`mJOr0tq9CU#9w@#i>Qb`pG$| zxv6<2`T<4xS*gh-#roN)MTy0_mH9dO#`>itnK{M!@$s2?nI-Y@dIgoYIBatBQ%ZAE L?LfvA12F>tQq&;w diff --git a/utils/__pycache__/bbox.cpython-36.pyc b/utils/__pycache__/bbox.cpython-36.pyc deleted file mode 100644 index d39c1a6980a0496e90ec16f1791691502076be45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2619 zcmaJ?OK%)S5bo}IuXp{*Cb3NfIRwO_5GxOfr$`h?2of#ffW#1Yh+yj)?*@>ZeNvZ?VSw45zY<83%1}9kk;-Y^6!6S~UOSJF?*&%)~}Nbo-3SO{~6#|qO7N{s}P^)l2?ZO3hBuQOK zQcp4%ZEq+#WcD{c<)GK_wLR{42io2@bli`|)Cc7)?X{99%i>HsNwgg&+R0j}iW{D` zvpCsNE)Gy+nDU7PL@Gq!3HFSs1+eEA*Y9Pq%GO&++>ZzO`ld>IaVyW(cViW0@9zV$ z%j=`On`G4yMG(f_@)zJ}2TLyyy6|FciyT28@gt<(#Gqh3 zCdIB$D>?50k)AazvBNwtu}VxzIBZqL4lM>l?MA8%9L!}JT+L(N+^8gERq1pC%?S)M z!h`V&pBD$qNA^)%m?f__@GQ0*8MuZXOzVj$Tg5Eg`x+B1Rl^=*ma3!of3_KatzyxU zZHuqkR=QMdY&2W~eh~HJFw~VW?5A>+P`?_64@Xf_O4Jf5C%x(<5%TddLa-l+!%*4-8UgQj6j?65?3fwkW z!^np0+P`Er$xNt=4Rx5B79ty$t8@o>tR6*47#-_xYcalvb_wk=+Lc!>qn8kihx^h; zu}Y%hTQE3V24URizCh!)SOm9)I*Zw(Yf>%{7Ur5y(ZLlC9o!ZLhpQ2fZ862wjRbkx zvyB8mP!AFd8E)0t1i5Wum#@$nG_qC@<`ZY)(kXNrB-n*kO1y~=_}n@xyuz=u)2skE zlOPAUi2vXXihT~R8=K2qHfd}UnsQk?J4qVljf%FBm=_Jbq~{DA8B#dhJ3i>>;7Fpy z9x&~o?E%-$CGCs`-E{COv_2(#*iA^WV|{ar`-x!3ax z7c`IooKulF5hqlp_9Q3*8c{p!iB)o4Wipq~^{UW^$^z#(y?HqQ*r>p)tgf(fwy408 z8r+C=|1D{Ywk_x7!V@v6!kUGG*4Jpy4xPP-wzMPi<-(hCS&@rRQ1`Iz1lCpC{BVBh ziDhQ%n00CY3%#)JCktSjPn0E zf7>KxdwuXT|HZj;bw>W3(Q`0cBSLQ04r1x@CDc_%=KI6A;b^PdZ#S)Bevt?w%G6pN z2gc|6XWR;t=fe-U6p{=IVcN_3 zN;qkMr<)|3X|k`q;mGJ{)BssOf^iOV3KiQmH7C=Rbh{Vk9TjIC?K~WHV;F}OIxzAO z_Lmipcbfs>6J}(sv)7e*N4uT4+wSDrRdM?@`CE}H+6#@oPheI?_%lAwZM=lb;6C&D zdo*HIypCs(+sphE=o#@YxXT5`Hm`zLg}R5m4SpA|jWZbahx;WcBF)rwNVIGGohiD% pDJJE!MK6TedhBg6CCSuh|I@OFeMXhQJTUZF`%pK(>M!`Ue*o;9O)dZc diff --git a/utils/__pycache__/colors.cpython-36.pyc b/utils/__pycache__/colors.cpython-36.pyc deleted file mode 100644 index ec9045f86b95574c4da37c2db7ace19f8857d784..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1685 zcmd6nJ8u&~5P4$s=YaK4Ln*GLXQ z6hKFTsGx#|0*Q)>j*1E@3WVHHAPS3)3emvK-s;Ftz?Q$+dC%-wx6t2Tn*8y36?z%_ z#T!-SVBkDps|<4Bf>(Euc_Te2fQz~ZJyxR6=tU#R(`$4WN=EK5_P(r6 zzpdxNfQ=hOIb{7ivp5fi(cXvksQ!qxqXwg>@3QvYS^OS5-(EY93q2S|@kNwlHh!O# z9_-KNP51L)9OEeCfX&;Baw4nmLEiYs?fe5)P8ylZ>hi4rl&yCV<)O@fIO{u!cABq% za+mEhlEod#;-*d9QL7KzJzY3r^EpZvX6!r%jNF;knab*vF>cn{UFfy(9?Y2hW0~HM z`jCw~Wak>Qa@NX;tiB67Fy4bv(`U}w=WQJAchvYx#=p`@-xB>-XAxUWGws}WZZSEkoh>%TR+!F#v+i!O`)GF<(|Jx+P;C@M;nzlGkL?L`4bBlN#TU0z@s01*C3O;Y&%ZgD_V7t>~zH2qq**_Qq-Ap_V<3=Lx6wHjOli!eGtUHBTBAbCOJDt2)>=lzmO!>XwUspmZzZ!`qPUc^g$-1a& zWiP0W4`;z|=`g|&dQhse(eCu(J**j(dMhf#ihWKoH{i$;p1YHXe{*%8lWmPKLYO9u z5grm=6Cwh={<;OiQ^Hq5AK?w*E#WfZ31NZop74usgK&@VjPQlfCR`+(Bb*|9BGd>! z3GWD!@PqK3@Q5%+cu5e14}_0|6NoG2oLN+QsF;h~27IZrPVr1LfmS5Wnj3n7rs81H I`Jcak0jE5DZ~y=R diff --git a/utils/__pycache__/tools.cpython-36.pyc b/utils/__pycache__/tools.cpython-36.pyc deleted file mode 100644 index d5cc22e896b4ca268a7ee20a46fd99326d418a94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9637 zcmcgyO^hSQb?)jvHh+f0nc>dP{%F-&t!0g}Kgvpx?ZraByY?!Q#)7oYO0+gi537f4 z4%yAAZtie|?n`EsC(U0waE>eZ`P@2_6-g@uL6+phKfH-BBz{#l#5Y}8-H9sfBBu5mWdTI$JK zx_mcU2G_YUFo#yl8rm(JQM)-P4V{*Qy2b5>TC2=U+`+TLFY+2+c*t5+UgwK|)Odq0 z;km$<`3jzO-sGq7T;!|#G@cE<#?Ron#Lx0`crNqv{0Tf)_$7XUKM71t?((MqImOxg z+WOZHK&iHIz)-I1x$}B74*BcRBzHD9qe(l6#_KG%{K+UtI@E_Qdg{2}z#Tg%5-rvC z2^-s3XTkymgV^MAO_Wgo^7^&!#lDEIbq4;>50h)}im2~*lK9%TFTD85UNneaxHe9L zL3}NVqCtFhw3j>WIOq_YNv-ka-_<_ULj419UeRpjdKBl*(A)NTAmY6A-S=+`5sBO!c}Xv~f*4?| zYyq>#C8>gzE{KE*?wAUV>8y@ljY;^=38ZO1edXtW`!{s`PacB?K5s}o8qrzd+A7f zEtHtLT9+_T-ANa!MEPP`PZu#$J*~-St)RB7)^^{`8fdAfjcuJf-1$V`($8qAlU7hR zZfZOKevdd&p|xa7lh@@+r(>^U>{7a<#__TosqmB)o_-apEl+VhVmnvwX*;*>X@Qn4 zgKvL5j}3hLVb%cUP&mxBB5QTjnzFWt+9_ViRxz$J&+q$RW|b+uSm){Ke5fy_tLajI zfjIpQEnDqs+1lib^z<$5{*SUVz*P@k?J!W<@%I5OZ6n)JNH75+&0F-_zqKsR07i3nl4+er`I zd(nUcsiNQzGLC)jVq_-@Ltb-vfbAH1qY)`eXW+%L z8+t>Z2D!u6-gVNMDQ4M8Eoc&Vpm^_sh!70b~pMIeK}WkPt>&`}pfK{kl1b zI^H0D`RZZExhn8l!V9~8UJb{?wp_Izuj^tNcyiW998;dD;GEZQqIrJ4^Z(O)N0$5l z!YoO?pglRpFF}Mqhx>oeC~=C=t)eK{BTiGDi8X>#m?mUEbZiDel5&Itvkj-7Qe0{o zOIcsrFE(8wQ=CuQQtGA~Vjw0xX+_t8saVm;Q_*(HsVZtJC0)~rtbw=b52njHi@qmK z^1@e4LQ9td{LFNc3u+RqiPoj#HgF^I>RR&?Te&%b>B`MLSx$!DgzDZzoB^g^Mh=C% z#hNz`OgS|Qx&$c!_-A^PAjSsK4jg-|$=%G&7zUzM8AKj$OFNg>yD(K0k+;EWKWLi-9-Ft2kr87qodWjL}PUi5qrjJdDS~yo8rvh>Eopgj-Qw z+QMAPOE2bzKZ^6p6qh$ze=4`6jh0fGFC6iKdF9Yw<&{GZlUvdosK;(zo>|&f9T7qM z(0sNQ>HTnpu~e~ES$MmOeXR=IGgqv`ipgNMsuCn6EN)dKtSy;#4%?Df=diVwVZ;Tz zO&dcxK@1+o`QjF=7a)Znj`MmV#`EvCvdgVSGGO!ADszN4RUNd=^~f2|or3>yYXt^B z=!QqSw5nkIY>P@k2p{2Ndq5sIk(Xz5=1Y%}UMQC`4`dl~XKEyu@D>Xc+S*xmR$pZ` zQlNpg40CZ8cz7Gb+Gg z7ML*BaM%UF>bm$2X6=v>rmKdVROb!caRYjSldQSVQiPkR8K|*y+O{U1OU=wmS!y;l zYRl}f0f?RIsR0Q3WY*-z2}^W3U(%87{*SU!YNpm2yx=+Dl=v**LrvE*CoAI+W*4o7 zY_-vf8d@vGk*#zF$FyD^AkJWRmC@4MCDUdzndRZ=8lKY@|gPKh@3v z_qHyQox7871EF{Pv!rG#J~er4MdpYlQ|2p7)FpW=jj7x#0; z23i}BdtefI3JxTLq+4t#k44l}k9pzJ(V$3>d!+PENA1GQ=l=;IEW)p|JZi(fBG7s) z_)^Zv79>7`+@_MIOH}(h6-3;+BYuf@bV#49 zB{tDZR^b-j@7qXYR8sq_mR0GwisaDb7wA)xqWiOK0b*B0s%C-PTO`7DByIF<_K1nq zN7~MR%vv>WgF5v1$jlbAMz)kKQ|jlmmfEnBi@?!9-9o*QLc!97rp8OBK`T~SLMnsS z1S^T;$+Xp&wJyt6SV`cVQlq#*6s;sqx`=cTEG}$bBV8_P1q`zmFx2WuLD)z22$ zK`@Qh0JZtYd$6`RqbzJ3kSrgNL69PQKtAKDaOs@_s)PT+CBBc^x~{;FJu8AD?Yu-s zaKt_`eJM!L^AZdn1!t7zjJ6UA`AEzV&P+x^PWfwu_*BbAWD<>G42uv&X_B_0=5q{+ucOe|GHmNISyx!q2D{Flh5EpG>vg@MS2PoDo!m97uF=%5>rGvd zH|fv{>6%(ysaih^1)g-7|&pGL2W53x%1hP1Hw0B=&7 zlpvL)z=%75Vz(Y#r$CD$bR{%qge?@8;IRTB0@M9!=54(|9iB@bi9LMG}f?dHulamE*njD1HMy4Xqg?ZNCQYe!A_kl-Jo-l0x(k>t({Z;cEu3<7*6 zNl1x8C?ywJMecWnf-w%m1JhfbTuMyJ7{h(KYN=75x`D z^44hudtZlNpN{-NM#c^^%CQR^vQyMxY^c^XvPGEn#^g?7f)-?m<&58i>4Fzemq=1z z+>oDDHF6j*Tn*&XVUmDj83veWvXC8a?EJ+&4F;?M*>e;WA1%swhcq!VUC5w=?htg> z!Dn0fbtzC%#-uPEkV3C3!=qfRbgY!BP=1znF{iz!Y)?B1+nZj}=}EPe5s~zL>nq|n zsGlt*C!UkC-yuZO29awPZ&C3!6$2ElGOUjp(yB_lO0txnt|q{L8}Q^wVfG>nQ4!KMDQ zK#$N6kEG!KULjQE%ON`09uhJ62J>)+yvX!j`rp}`rCK2{2=Dp~H+gO~QsRM^wvm{h zj+U@}>T4Y7D`(*rg>v)r?(2vcsOM44l7?fLCF}&ulENdHB}8I2i4G`>0W(FjAS9J_ zf*)JQY*kZ!OjoJ+F@2lyqo7WQhXvLqI0!>dGf-(JI05y^I9jR|p)Fz%X*Nn8zJzKr z+4;t=n2;BnU3{$T1)c3{qrEFzUX0Z1710Spm7;vt6UbfyrgEHDW|y4AyWu{&aoK%w z-3__SXiy$jo^E9GCYs;+>gM_6X@OS`n;xbW^0$s4FWP4LZz=jR{k=2x#hywUNGuz3 zeu4ajRBzbPr%+!n6%hndMwAhe{y;oWz3?$wlfFZ|AUo#F6CKo$2NJ%Q>%neI4|*+q z?Bz8a+rSu==09ed#bt~sV>9AqRJ7`=dJ7+~;l}mff&+JS**?=J(j|(Fq(1#6D%fYp zGR!FqUzd?nRXQSJvPHs-$qhuiMjuiL<7mQV8Zg$EGLk`gNFq4&Q686|A5uow5V`K$ zgqfssFI+aa$fTlX!MUThEnPT_EMcM3IpmsRN2*C}mRJZvQ7@#>A{XtwiN(5asC<;n z@)x-bcb49`TzpKTPXH?SAvbQ;?jqm+V455MKagK31}A-0Qj_1HPXKohX_2$L&r&9Q z{qW<=jJo1W0mjOlk|Op<_s2g65cbE|;CeB>7RkIFr_UWHsa+Q`Ag`kGi&vC)0*|!~ z8Rkr59T^E%@%4$m%!Ig=ha#&k1ven63{j%! zQnewHga8>EcX1wewtas@$;)-K2-iFc>)XI6QWWck47Q~bw+z4iSRVVg(diYUn5fZM z6V6N!tYgwUE()_MxTUK71eM7@r4-@(U1o9!GhfK;$p;BV%ZQt0Byxw6P@o)%qSQbp z!c)7q?qc8n%2*ntVgw&RfUJD%k`gM-n2BM@zUfVBW zE*oBIBMfnR`J0$>{#)=l}K(?`*zX1l00SW~jJ& zpZb#Qf&cUnhk!*TZ_g0)eH6Jh60nPqw?v2@0m_iG1Yd^+zRGdQ^p;F?QI0DwA&|N2 zPZF7~&zs-=?#A8ro!jrdf9Ljx?f2h&=iRq&e;@;TWHV$4Oy{xOR+(TGMA2C*FH1<9 zW~G#{6zOB)oL4yIn5E0OO){cFp9)T2CXyHQhZB=Mr#lFKY&wtOjSRN<4EUfri-cAd z4XRQ4RTfGg;aMcdR@I^{~+ZlTvPEUJ- zt=W^q4a6S!fQTD>0-w@XF8CKXQPs20C`YE%Rn=Y9)m7D1HP_p1f8)IU{bQeyzsZTO z0rU-+HotE9+v^Nxt8i+keE1KkdVEbhjfC;1o-1rKKT<7}3iRiEhwaCB=} zbcpvIehi2kFw2hs2!dXRj8o2D5b(UgEx=Z7b7*syd$4wR6YTbN z@K7Y{NM&7s7*1`eVC_r^VCw**R(C}y$^F9{C^1sj>}Mlo?K#Loq0Zshf{4yh?aJW(Wn$kvwWugku0*od^8eLx8gL7o|~wa zsG=0W((~_QIaPriULqn zz6pTPfS$v_dNg1@>##0!=<@u@bl5sPb(9wvH>@|47id&PCXg8y07ppEdVSIC|HL*i z{&c6$bp`ptQr_4H*Ix?&cg6Nwu z>`!>lefs0?3Yx)+-U9l`Z+mQ?E=CodKv^hA-W3?V!o-CnK3+qrG2WiK@}-eo&7MUtuV=2oH@-R=AoxC`GxZjX{ z6!`$bhXDE=Z9~P4g>H_8iqd!iHQa!T*Y->%xssb8q^+4OpcSLUfmzxvpT;vG-$!-} z8o?2xkwVXKr7(PLrAetweM9XEE_*2Ne;227aa+nl_SdZNSvjP9YS$SXeqKxmg* zFgwhnU3QN7)C}keU3!ULVwVl?5;d51=yLN^L7vn(#23I1G4=~nlGt@Qbcd*Za<*e$ zt4~d10E9i^Ra0#Up-|>H>LGO5J$0*L@AO^lZh4%EDAIluK^W&L(m@nGo5yL*kyr5o zd@Fei!EFS`p)u4I+{3>f&JuqN9`(AO<-KEe`)_D3mg89r&!|~ZYCE52vqhaBf$70p z=9x-~=1R;zi5*PZ2pj}10Np()yOjw!7SjBNn#euSWU`@@7&mzy_Z!3AYsQY(nXSFg VYQI0nsF#@kU^|Q+JXU~I<3Ho8P0auR From a1f2ebb86f1f519439bf371f0e6ba0f3ef07d63b Mon Sep 17 00:00:00 2001 From: kcw Date: Tue, 29 May 2018 11:17:45 +0800 Subject: [PATCH 09/15] add readme --- README.md | 12 + model/yolo3.py | 467 ++++++++++++++++++++++++++- req.txt | 114 ------- requirements.txt | 30 ++ yolo3_one_file_to_detect_them_all.py | 18 +- 5 files changed, 514 insertions(+), 127 deletions(-) delete mode 100644 req.txt create mode 100644 requirements.txt diff --git a/README.md b/README.md index 0875cfcba..85442389a 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,20 @@ VOC (20 classes) (http://host.robots.ox.ac.uk/pascal/VOC/voc2012/) | 72% | check Grab the pretrained weights of yolo3 from https://pjreddie.com/media/files/yolov3.weights. +```wget -c https://pjreddie.com/media/files/yolov3.weights``` + +Environment setup: + +```pip install -r requirements.txt``` + +Detection on single image: + ```python yolo3_one_file_to_detect_them_all.py -w yolo3.weights -i dog.jpg``` +If your webcam is available: + +```python yolo3_cam.py -w yolov3.weights``` + ## Training ### 1. Data preparation diff --git a/model/yolo3.py b/model/yolo3.py index 79d90ca22..bd02427af 100644 --- a/model/yolo3.py +++ b/model/yolo3.py @@ -1,8 +1,18 @@ +import os +import multiprocessing import numpy as np from keras.layers import Conv2D, Input, BatchNormalization, LeakyReLU, ZeroPadding2D, UpSampling2D from keras.layers.merge import add, concatenate from keras.models import Model +import keras +import keras.backend as K +import keras.layers as KL +import keras.engine as KE +import keras.models as KM +from keras.optimizers import Adam + + def _conv_block(inp, convs, skip=True): x = inp count = 0 @@ -24,6 +34,27 @@ def _conv_block(inp, convs, skip=True): return add([skip_connection, x]) if skip else x +def conv_block(inp, convs, skip=True): + x = inp + count = 0 + + for conv in convs: + if count == (len(convs) - 2) and skip: + skip_connection = x + count += 1 + + if conv['stride'] > 1: x = ZeroPadding2D(((1,0),(1,0)))(x) # peculiar padding as darknet prefer left and top + x = Conv2D(conv['filter'], + conv['kernel'], + strides=conv['stride'], + padding='valid' if conv['stride'] > 1 else 'same', # peculiar padding as darknet prefer left and top + name='conv_' + str(conv['layer_idx']), + use_bias=False if conv['bnorm'] else True)(x) + if conv['bnorm']: x = BatchNormalization(epsilon=0.001, name='bnorm_' + str(conv['layer_idx']))(x) + if conv['leaky']: x = LeakyReLU(alpha=0.1, name='leaky_' + str(conv['layer_idx']))(x) + merged_graph = add([skip_connection, x]) + return merged_graph if skip else x + def _interval_overlap(interval_a, interval_b): x1, x2 = interval_a x3, x4 = interval_b @@ -56,6 +87,196 @@ def bbox_iou(box1, box2): return float(intersect) / union +class YoloLayer(): + def __init__(self, anchors, max_grid, batch_size, warmup_batches, ignore_thresh, + grid_scale, obj_scale, noobj_scale, xywh_scale, class_scale, + **kwargs): + # make the model settings persistent + self.ignore_thresh = ignore_thresh + self.warmup_batches = warmup_batches + self.anchors = tf.constant(anchors, dtype='float', shape=[1,1,1,3,2]) + self.grid_scale = grid_scale + self.obj_scale = obj_scale + self.noobj_scale = noobj_scale + self.xywh_scale = xywh_scale + self.class_scale = class_scale + + # make a persistent mesh grid + max_grid_h, max_grid_w = max_grid + + cell_x = tf.to_float(tf.reshape(tf.tile(tf.range(max_grid_w), [max_grid_h]), (1, max_grid_h, max_grid_w, 1, 1))) + cell_y = tf.transpose(cell_x, (0,2,1,3,4)) + self.cell_grid = tf.tile(tf.concat([cell_x,cell_y],-1), [batch_size, 1, 1, 3, 1]) + + super(YoloLayer, self).__init__(**kwargs) + + def build(self, input_shape): + super(YoloLayer, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, x): + input_image, y_pred, y_true, true_boxes = x + + # adjust the shape of the y_predict [batch, grid_h, grid_w, 3, 4+1+nb_class] + y_pred = tf.reshape(y_pred, tf.concat([tf.shape(y_pred)[:3], tf.constant([3, -1])], axis=0)) + + # initialize the masks + object_mask = tf.expand_dims(y_true[..., 4], 4) + + # the variable to keep track of number of batches processed + batch_seen = tf.Variable(0.) + + # compute grid factor and net factor + grid_h = tf.shape(y_true)[1] + grid_w = tf.shape(y_true)[2] + grid_factor = tf.reshape(tf.cast([grid_w, grid_h], tf.float32), [1,1,1,1,2]) + + net_h = tf.shape(input_image)[1] + net_w = tf.shape(input_image)[2] + net_factor = tf.reshape(tf.cast([net_w, net_h], tf.float32), [1,1,1,1,2]) + + """ + Adjust prediction + """ + pred_box_xy = (self.cell_grid[:,:grid_h,:grid_w,:,:] + tf.sigmoid(y_pred[..., :2])) # sigma(t_xy) + c_xy + pred_box_wh = y_pred[..., 2:4] # t_wh + pred_box_conf = tf.expand_dims(tf.sigmoid(y_pred[..., 4]), 4) # adjust confidence + pred_box_class = y_pred[..., 5:] # adjust class probabilities + + """ + Adjust ground truth + """ + true_box_xy = y_true[..., 0:2] # (sigma(t_xy) + c_xy) + true_box_wh = y_true[..., 2:4] # t_wh + true_box_conf = tf.expand_dims(y_true[..., 4], 4) + true_box_class = tf.argmax(y_true[..., 5:], -1) + + """ + Compare each predicted box to all true boxes + """ + # initially, drag all objectness of all boxes to 0 + conf_delta = pred_box_conf - 0 + + # then, ignore the boxes which have good overlap with some true box + true_xy = true_boxes[..., 0:2] / grid_factor + true_wh = true_boxes[..., 2:4] / net_factor + + true_wh_half = true_wh / 2. + true_mins = true_xy - true_wh_half + true_maxes = true_xy + true_wh_half + + pred_xy = tf.expand_dims(pred_box_xy / grid_factor, 4) + pred_wh = tf.expand_dims(tf.exp(pred_box_wh) * self.anchors / net_factor, 4) + + pred_wh_half = pred_wh / 2. + pred_mins = pred_xy - pred_wh_half + pred_maxes = pred_xy + pred_wh_half + + intersect_mins = tf.maximum(pred_mins, true_mins) + intersect_maxes = tf.minimum(pred_maxes, true_maxes) + + intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.) + intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] + + true_areas = true_wh[..., 0] * true_wh[..., 1] + pred_areas = pred_wh[..., 0] * pred_wh[..., 1] + + union_areas = pred_areas + true_areas - intersect_areas + iou_scores = tf.truediv(intersect_areas, union_areas) + + best_ious = tf.reduce_max(iou_scores, axis=4) + conf_delta *= tf.expand_dims(tf.to_float(best_ious < self.ignore_thresh), 4) + + """ + Compute some online statistics + """ + true_xy = true_box_xy / grid_factor + true_wh = tf.exp(true_box_wh) * self.anchors / net_factor + + true_wh_half = true_wh / 2. + true_mins = true_xy - true_wh_half + true_maxes = true_xy + true_wh_half + + pred_xy = pred_box_xy / grid_factor + pred_wh = tf.exp(pred_box_wh) * self.anchors / net_factor + + pred_wh_half = pred_wh / 2. + pred_mins = pred_xy - pred_wh_half + pred_maxes = pred_xy + pred_wh_half + + intersect_mins = tf.maximum(pred_mins, true_mins) + intersect_maxes = tf.minimum(pred_maxes, true_maxes) + intersect_wh = tf.maximum(intersect_maxes - intersect_mins, 0.) + intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1] + + true_areas = true_wh[..., 0] * true_wh[..., 1] + pred_areas = pred_wh[..., 0] * pred_wh[..., 1] + + union_areas = pred_areas + true_areas - intersect_areas + iou_scores = tf.truediv(intersect_areas, union_areas) + iou_scores = object_mask * tf.expand_dims(iou_scores, 4) + + count = tf.reduce_sum(object_mask) + count_noobj = tf.reduce_sum(1 - object_mask) + detect_mask = tf.to_float((pred_box_conf*object_mask) >= 0.5) + class_mask = tf.expand_dims(tf.to_float(tf.equal(tf.argmax(pred_box_class, -1), true_box_class)), 4) + recall50 = tf.reduce_sum(tf.to_float(iou_scores >= 0.5 ) * detect_mask * class_mask) / (count + 1e-3) + recall75 = tf.reduce_sum(tf.to_float(iou_scores >= 0.75) * detect_mask * class_mask) / (count + 1e-3) + avg_iou = tf.reduce_sum(iou_scores) / (count + 1e-3) + avg_obj = tf.reduce_sum(pred_box_conf * object_mask) / (count + 1e-3) + avg_noobj = tf.reduce_sum(pred_box_conf * (1-object_mask)) / (count_noobj + 1e-3) + avg_cat = tf.reduce_sum(object_mask * class_mask) / (count + 1e-3) + + """ + Warm-up training + """ + batch_seen = tf.assign_add(batch_seen, 1.) + + true_box_xy, true_box_wh, xywh_mask = tf.cond(tf.less(batch_seen, self.warmup_batches+1), + lambda: [true_box_xy + (0.5 + self.cell_grid[:,:grid_h,:grid_w,:,:]) * (1-object_mask), + true_box_wh + tf.zeros_like(true_box_wh) * (1-object_mask), + tf.ones_like(object_mask)], + lambda: [true_box_xy, + true_box_wh, + object_mask]) + + """ + Compare each true box to all anchor boxes + """ + wh_scale = tf.exp(true_box_wh) * self.anchors / net_factor + wh_scale = tf.expand_dims(2 - wh_scale[..., 0] * wh_scale[..., 1], axis=4) # the smaller the box, the bigger the scale + + xy_delta = xywh_mask * (pred_box_xy-true_box_xy) * wh_scale * self.xywh_scale + wh_delta = xywh_mask * (pred_box_wh-true_box_wh) * wh_scale * self.xywh_scale + conf_delta = object_mask * (pred_box_conf-true_box_conf) * self.obj_scale + (1-object_mask) * conf_delta * self.noobj_scale + class_delta = object_mask * \ + tf.expand_dims(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=true_box_class, logits=pred_box_class), 4) * \ + self.class_scale + + loss_xy = tf.reduce_sum(tf.square(xy_delta), list(range(1,5))) + loss_wh = tf.reduce_sum(tf.square(wh_delta), list(range(1,5))) + loss_conf = tf.reduce_sum(tf.square(conf_delta), list(range(1,5))) + loss_class = tf.reduce_sum(class_delta, list(range(1,5))) + + loss = loss_xy + loss_wh + loss_conf + loss_class + + loss = tf.Print(loss, [grid_h, avg_obj], message='avg_obj \t\t', summarize=1000) + loss = tf.Print(loss, [grid_h, avg_noobj], message='avg_noobj \t\t', summarize=1000) + loss = tf.Print(loss, [grid_h, avg_iou], message='avg_iou \t\t', summarize=1000) + loss = tf.Print(loss, [grid_h, avg_cat], message='avg_cat \t\t', summarize=1000) + loss = tf.Print(loss, [grid_h, recall50], message='recall50 \t', summarize=1000) + loss = tf.Print(loss, [grid_h, recall75], message='recall75 \t', summarize=1000) + loss = tf.Print(loss, [grid_h, count], message='count \t', summarize=1000) + loss = tf.Print(loss, [grid_h, tf.reduce_sum(loss_xy), + tf.reduce_sum(loss_wh), + tf.reduce_sum(loss_conf), + tf.reduce_sum(loss_class)], message='loss xy, wh, conf, class: \t', summarize=1000) + + + return loss*self.grid_scale + + def compute_output_shape(self, input_shape): + return [(None, 1)] + def make_yolov3_model(): input_image = Input(shape=(None, None, 3)) @@ -153,8 +374,246 @@ def make_yolov3_model(): model = Model(input_image, [yolo_82, yolo_94, yolo_106]) return model +def dummy_loss(y_true, y_pred): + return tf.sqrt(tf.reduce_sum(y_pred)) + class YOLO3(): - ''' - To be continued - ''' - pass \ No newline at end of file + """ + Encapsulates the YOLO3 model functionality. + The actual Keras model is in the keras_model property. + """ + def __init__(self, mode, config, model_dir): + """ + mode: Either "training" or "inference" + config: A Sub-class of the Config class + model_dir: Directory to save training logs and trained weights + """ + assert mode in ['training', 'inference'] + self.mode = mode + self.config = config + self.model_dir = model_dir + self.set_log_dir() + self.keras_model = self.build(mode=mode, config=config) + + def build(self, mode, config): + """ + Build YOLO3 architecture. + mode: Either ["training", "inference"] + Config: {nb_class,anchors,max_box_per_image, + max_grid,batch_size,warmup_batches,ignore_thresh, + grid_scales,obj_scale,noobj_scale,xywh_scale,class_scale} + """ + assert mode in ['training', 'inference'] + + input_image = KL.Input(shape=(None, None, 3)) # net_h, net_w, 3 + true_boxes = KL.Input( + shape=(1, 1, 1,config.max_box_per_image, 4)) + true_yolo_1 = KL.Input( + shape=(None, None, len(config.anchors)//6, 4+1+config.nb_class)) + # grid_h, grid_w, nb_anchor, 5+nb_class + true_yolo_2 = KL.Input( + shape=(None, None, len(config.anchors)//6, 4+1+config.nb_class)) + # grid_h, grid_w, nb_anchor, 5+nb_class + true_yolo_3 = KL.Input( + shape=(None, None, len(config.anchors)//6, 4+1+config.nb_class)) + # grid_h, grid_w, nb_anchor, 5+nb_class + # Layer 0 => 4 + x = _conv_block(input_image, [ + {'filter': 32, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 0}, + {'filter': 64, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 1}, + {'filter': 32, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 2}, + {'filter': 64, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 3}]) + # Layer 5 => 8 + x = _conv_block(x, [ + {'filter': 128, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 5}, + {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 6}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 7}]) + # Layer 9 => 11 + x = _conv_block(x, [ + {'filter': 64, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 9}, + {'filter': 128, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 10}]) + # Layer 12 => 15 + x = _conv_block(x, [ + {'filter': 256, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 12}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 13}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 14}]) + # Layer 16 => 36 + for i in range(7): + x = _conv_block(x, [ + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 16+i*3}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 17+i*3}]) + skip_36 = x + # Layer 37 => 40 + x = _conv_block(x, [ + {'filter': 512, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 37}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 38}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 39}]) + # Layer 41 => 61 + for i in range(7): + x = _conv_block(x, [ + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 41+i*3}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 42+i*3}]) + skip_61 = x + # Layer 62 => 65 + x = _conv_block(x, [ + {'filter': 1024, 'kernel': 3, 'stride': 2, 'bnorm': True, 'leaky': True, 'layer_idx': 62}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 63}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 64}]) + # Layer 66 => 74 + for i in range(3): + x = _conv_block(x, [ + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 66+i*3}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 67+i*3}]) + # Layer 75 => 79 + x = _conv_block(x, [ + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 75}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 76}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 77}, + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 78}, + {'filter': 512, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 79} + ], do_skip=False) + # Layer 80 => 82 + pred_yolo_1 = _conv_block(x, [ + {'filter': 1024, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 80}, + {'filter': (3*(5+nb_class)), 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 81} + ], do_skip=False) + # YOLO Layer + loss_yolo_1 = YoloLayer( + anchors[12:], + [1*num for num in max_grid], + batch_size, + warmup_batches, + ignore_thresh, + grid_scales[0], + obj_scale, + noobj_scale, + xywh_scale, + class_scale)( + [input_image, pred_yolo_1, true_yolo_1, true_boxes]) + # Layer 83 => 86 + x = _conv_block(x, [ + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 84} + ], do_skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_61]) + + # Layer 87 => 91 + x = _conv_block(x, [ + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 87}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 88}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 89}, + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 90}, + {'filter': 256, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 91} + ], do_skip=False) + + # Layer 92 => 94 + pred_yolo_2 = _conv_block(x, [ + {'filter': 512, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 92}, + {'filter': (3*(5+nb_class)), 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 93} + ], do_skip=False) + loss_yolo_2 = YoloLayer( + anchors[6:12], + [2*num for num in max_grid], + batch_size, + warmup_batches, + ignore_thresh, + grid_scales[1], + obj_scale, + noobj_scale, + xywh_scale, + class_scale)( + [input_image, pred_yolo_2, true_yolo_2, true_boxes]) + # Layer 95 => 98 + x = _conv_block(x, [ + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 96} + ], do_skip=False) + x = UpSampling2D(2)(x) + x = concatenate([x, skip_36]) + # Layer 99 => 106 + pred_yolo_3 = _conv_block(x, [ + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 99}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 100}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 101}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 102}, + {'filter': 128, 'kernel': 1, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 103}, + {'filter': 256, 'kernel': 3, 'stride': 1, 'bnorm': True, 'leaky': True, 'layer_idx': 104}, + {'filter': (3*(5+nb_class)), 'kernel': 1, 'stride': 1, 'bnorm': False, 'leaky': False, 'layer_idx': 105} + ], do_skip=False) + loss_yolo_3 = YoloLayer( + anchors[:6], + [4*num for num in max_grid], + batch_size, + warmup_batches, + ignore_thresh, + grid_scales[2], + obj_scale, + noobj_scale, + xywh_scale, + class_scale)( + [input_image, pred_yolo_3, true_yolo_3, true_boxes]) + + if mode == "training": + model = KM.Model( + [input_image, true_boxes, true_yolo_1, true_yolo_2, true_yolo_3], + [loss_yolo_1, loss_yolo_2, loss_yolo_3]) + elif mode == "inference": + model = KM.Model( + input_image, + [pred_yolo_1, pred_yolo_2, pred_yolo_3]) + # Muti-GPU Model Build Here + if config.GPU_COUNT > 1: + pass + return model + + def compile(self, lr): + assert self.mode in ['training'] + optimizer = Adam(lr=lr, clipnorm=0.001) + self.model.compile(loss=self.dummy_loss, optimizer=optimizer) + + def train(self, train_dataset, val_dataset, + learning_rate, epochs, layers,augmentation=None): + assert self.mode == "training", "train" + # Callbacks + callbacks = [ + keras.callbacks.TensorBoard( + log_dir=self.log_dir, histogram_freq=0, write_graph=True, write_images=False), + keras.callbacks.ModelCheckpoint( + self.checkpoint_path, verbose=0, save_weights_only=True) + ] + # Data generators + ################ data_generator need to be impl + ''' + train_generator = data_generator(train_dataset, + self.config, shuffle=True, + augmentation=augmentation, batch_size=self.config.BATCH_SIZE) + val_generator = data_generator(val_dataset, + self.config, shuffle=True, batch_size=self.config.BATCH_SIZE) + ''' + # Train + # log("\nStarting at epoch {}. LR={}\n".format(self.epoch, learning_rate)) + # log("Checkpoint Path: {}".format(self.checkpoint_path)) + # self.set_trainable(layers) + self.compile(self.config.learning_rate) + if os.name is 'nt': + workers = 0 + else: + workers = multiprocessing.cpu_count() + self.keras_model.fit_generator( + train_generator, + steps_per_epoch = len(train_generator) * config['train']['train_times'], + epochs = config['train']['nb_epochs'] + config['train']['warmup_epochs'], + verbose = 2 if config['train']['debug'] else 1, + callbacks = callbacks, + workers = workers, + max_queue_size = 100, + use_multiprocessing=True) + self.epoch = max(self.epoch, epochs) + + def detect(self): + pass + def trainingModel(self): + pass + def inferenceModel(self): + pass + def dummy_loss(y_true, y_pred): + return tf.sqrt(tf.reduce_sum(y_pred)) diff --git a/req.txt b/req.txt deleted file mode 100644 index 485e8747c..000000000 --- a/req.txt +++ /dev/null @@ -1,114 +0,0 @@ -absl-py==0.1.12 -amqp==2.2.2 -args==0.1.0 -asn1crypto==0.24.0 -astor==0.6.2 -billiard==3.5.0.3 -bleach==1.5.0 -celery==4.1.0 -certifi==2018.1.18 -cffi==1.11.5 -chardet==3.0.4 -click==6.7 -clint==0.5.1 -crontab==0.22.0 -cryptography==2.2.1 -cycler==0.10.0 -Cython==0.28.1 -decorator==4.2.1 -dev==0.4.0 -docopt==0.6.2 -entrypoints==0.2.3 -enum34==1.1.6 -et-xmlfile==1.0.1 -Flask==0.12.2 -Flask-Script==2.0.6 -futures==3.1.1 -gast==0.2.0 -gevent==1.2.2 -GPUtil==1.2.3 -greenlet==0.4.13 -grpcio==1.10.0 -gunicorn==19.7.1 -h5py==2.7.1 -html5lib==0.9999999 -idna==2.6 -ipykernel==4.8.2 -ipython==6.2.1 -ipython-genutils==0.2.0 -ipywidgets==7.1.2 -itsdangerous==0.24 -jdcal==1.3 -jedi==0.11.1 -Jinja2==2.10 -jsonschema==2.6.0 -jupyter-client==5.2.3 -jupyter-core==4.4.0 -Keras==2.1.5 -kiwisolver==1.0.1 -kombu==4.1.0 -Markdown==2.6.11 -MarkupSafe==1.0 -matplotlib==2.2.2 -mistune==0.8.3 -nbconvert==5.3.1 -nbformat==4.4.0 -networkx==2.1 -notebook==5.4.1 -numpy==1.14.2 -odfpy==1.3.6 -opencv-python==3.4.0.12 -openpyxl==2.5.1 -pandocfilters==1.4.2 -parso==0.1.1 -pbr==4.0.2 -pexpect==4.4.0 -pickleshare==0.7.4 -Pillow==5.0.0 -prompt-toolkit==1.0.15 -protobuf==3.5.2.post1 -ptyprocess==0.5.2 -pycocotools==2.0 -pycosat==0.6.3 -pycparser==2.18 -Pygments==2.2.0 -pyOpenSSL==17.5.0 -pyparsing==2.2.0 -PySocks==1.6.8 -python-crontab==2.2.8 -python-dateutil==2.7.2 -pytz==2018.3 -PyWavelets==0.5.2 -PyYAML==3.12 -pyzmq==17.0.0 -redis==2.10.6 -requests==2.18.4 -ruamel.yaml==0.15.37 -scikit-image==0.13.1 -scikit-learn==0.19.1 -scipy==1.0.1 -Send2Trash==1.5.0 -simplegeneric==0.8.1 -six==1.11.0 -stevedore==1.28.0 -tablib==0.12.1 -tensorboard==1.6.0 -tensorflow==1.6.0 -tensorflow-tensorboard==1.5.1 -termcolor==1.1.0 -terminado==0.8.1 -testpath==0.3.1 -tornado==5.0.1 -traitlets==4.3.2 -unicodecsv==0.14.1 -urllib3==1.22 -vine==1.1.4 -virtualenv==15.2.0 -virtualenv-clone==0.3.0 -virtualenvwrapper==4.8.2 -wcwidth==0.1.7 -webencodings==0.5.1 -Werkzeug==0.14.1 -widgetsnbextension==3.1.4 -xlrd==1.1.0 -xlwt==1.3.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..0eb506f9c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +absl-py==0.2.0 +astor==0.6.2 +bleach==1.5.0 +certifi==2018.4.16 +cycler==0.10.0 +gast==0.2.0 +grpcio==1.11.0 +h5py==2.7.1 +html5lib==0.9999999 +Keras==2.1.5 +kiwisolver==1.0.1 +Markdown==2.6.11 +matplotlib==2.2.2 +numpy==1.14.2 +opencv-python==3.4.0.12 +pandas==0.23.0 +Pillow==5.1.0 +protobuf==3.5.2.post1 +pyparsing==2.2.0 +python-dateutil==2.7.2 +pytz==2018.4 +PyYAML==3.12 +scipy==1.0.1 +seaborn==0.8.1 +selenium==3.12.0 +six==1.11.0 +tensorboard==1.7.0 +tensorflow==1.7.0 +termcolor==1.1.0 +Werkzeug==0.14.1 diff --git a/yolo3_one_file_to_detect_them_all.py b/yolo3_one_file_to_detect_them_all.py index 4d59fd7cb..248c78b5b 100644 --- a/yolo3_one_file_to_detect_them_all.py +++ b/yolo3_one_file_to_detect_them_all.py @@ -38,15 +38,15 @@ def _main_(args): obj_thresh, nms_thresh = 0.5, 0.45 anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ - "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ - "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ - "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ - "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ - "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ - "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ - "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ - "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ + "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ + "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ + "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ + "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] # make the yolov3 model to predict 80 classes on COCO yolov3 = make_yolov3_model() From 690ac5702e94a8d14b749c1569b16787b479479e Mon Sep 17 00:00:00 2001 From: kcw Date: Tue, 29 May 2018 11:20:09 +0800 Subject: [PATCH 10/15] _ --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 85442389a..b35140441 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,11 @@ Environment setup: ```pip install -r requirements.txt``` +If Nvidia GPU is available: + +```pip uninstall tensorflow``` +```pip install tensorflow-gpu``` + Detection on single image: ```python yolo3_one_file_to_detect_them_all.py -w yolo3.weights -i dog.jpg``` From 768a8ee6fd4acefd3e3dce9f642b28cedd83f7dd Mon Sep 17 00:00:00 2001 From: kcw Date: Tue, 29 May 2018 11:20:48 +0800 Subject: [PATCH 11/15] _ --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b35140441..b698777cf 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Environment setup: If Nvidia GPU is available: ```pip uninstall tensorflow``` + ```pip install tensorflow-gpu``` Detection on single image: From 53b5ad9a983d68442424cb03d55abb516a62c541 Mon Sep 17 00:00:00 2001 From: kcw Date: Mon, 4 Jun 2018 18:17:38 +0800 Subject: [PATCH 12/15] _ --- RMS_yolo3.py | 210 ++++++++++++++++++++++++++++++++++++++++++++++++ config.py | 83 +++++++++++++++++++ yolo3_cam.py | 5 +- yolo3_cam_mp.py | 106 ++++++++++++++++++++++++ 4 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 RMS_yolo3.py create mode 100644 config.py create mode 100644 yolo3_cam_mp.py diff --git a/RMS_yolo3.py b/RMS_yolo3.py new file mode 100644 index 000000000..ba92303d8 --- /dev/null +++ b/RMS_yolo3.py @@ -0,0 +1,210 @@ +import base64 +import os +import sys +import argparse +import warnings +warnings.filterwarnings("ignore") +os.environ.setdefault('PATH', '') +import numpy as np +import redis +import time +import json +from io import BytesIO +from multiprocessing import Process, Pipe, current_process, Lock +import GPUtil +from skimage.measure import find_contours +import struct +import cv2 +import numpy as np +import config + +# connect to Redis server +redispool = redis.ConnectionPool(host=config.REDIS_HOST, + port=config.REDIS_PORT, + db=config.REDIS_DB, + socket_keepalive=True) + +try: + print('Testing Redis Connection') + redisdbSession = redis.StrictRedis(connection_pool=redispool) + response = redisdbSession.client_list() + print('Redis Connection Established') +except redis.ConnectionError as e: + print(e) + sys.exit(1) + +np.set_printoptions(threshold=np.nan) +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="0" + +# set some parameters +net_h, net_w = 416, 416 +obj_thresh, nms_thresh = 0.7, 0.7 +anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] +labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ + "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ + "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ + "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ + "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] + +class mlWorker(Process): + def __init__(self, LOCK, GPU="", FRAC=0): + Process.__init__(self) + self.lock = LOCK + if GPU: + print('{} using GPUid: {}, Name: {}'.format(self.name, str(GPU.id), str(GPU.name))) + os.environ["CUDA_VISIBLE_DEVICES"] = str(GPU.id) + self.device = '/device:GPU:0' + else: + self.device = '' + self.GPU = GPU + self.frac = FRAC + self.counter = 0 + self.dt = 0.0 + + def run(self): + from utils.weightreader import WeightReader + from utils.bbox import BoundBox + from utils.tools import preprocess_input, decode_netout + from utils.tools import correct_yolo_boxes, do_nms, draw_boxes + from model.yolo3 import make_yolov3_model + import tensorflow as tf + from PIL import Image + self.Image = Image + self.preprocess_input = preprocess_input + self.decode_netout = decode_netout + self.correct_yolo_boxes = correct_yolo_boxes + self.do_nms = do_nms + self.draw_boxes = draw_boxes + if self.GPU: + print('ML Process: {} starting, using GPU: {}, frac: {}'.format(self.name,self.GPU.id,self.frac)) + keras.backend.clear_session() + conf = tf.ConfigProto() + conf.gpu_options.per_process_gpu_memory_fraction = self.frac + set_session(tf.Session(config=conf)) + # make the yolov3 model to predict 80 classes on COCO + _model = make_yolov3_model() + + # load the weights trained on COCO into the model + weight_reader = WeightReader(config.MODEL_PATH) + weight_reader.load_weights(_model) + + graph = tf.get_default_graph() + print('ML Process: {} started'.format(self.name)) + self.mainloop(model=_model, graph=graph) + + def mainloop(self, model='', graph=''): + while True: + # attempt to grab a batch of images from the database, then + # initialize the image IDs and batch of images themselves + try: + redisdbSession = redis.StrictRedis(connection_pool=redispool) + self.lock.acquire() + query = redisdbSession.lrange(config.IMAGE_QUEUE, 0, config.BATCH_SIZE - 1) + redisdbSession.ltrim(config.IMAGE_QUEUE, len(query), -1) + self.lock.release() + imageIDs = [] + thresholds = {} + batch = [] + # loop over the queue + # deserialize the object and obtain the input image + if query: + for item in query: + data = json.loads(item) + image = self.base64_decode_image(data["image"]) + image = self.preprocess_input(image, net_h, net_w) + # check to see if the batch list is None + batch.append(image) + # update the list of image IDs + imageIDs.append(data["id"]) + thresholds[data["id"]] = data["threshold"] + + # check to see if we need to process the batch + if len(imageIDs) > 0: + #print('{}: Procesing {} images!'.format(self.name, len(imageIDs))) + start = time.time() + with graph.as_default(): + results = model.predict(batch[0]) + end = time.time() + et = end - start + self.dt += float(et) + self.counter += 1 + adt = float(self.dt)/float(self.counter) + print('avg dt: %f' % adt) + # loop over the image IDs and their corresponding set of + # results from our model + output = [] + output = self.extract_result(results, + throttle=float(thresholds[imageID])) + redisdbSession.set(imageID, json.dumps(output)) + # sleep for a small amount + time.sleep(config.SERVER_SLEEP*2) + except Exception as e: + print(e) + time.sleep(config.SERVER_SLEEP) + continue + + def extract_result(self, results, throttle='0.95'): + boxes = [] + + for i in range(len(yolos)): + # decode the output of the network + boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) + # correct the sizes of the bounding boxes + correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) + # suppress non-maximal boxes + do_nms(boxes, nms_thresh) + + return output + + def base64_decode_image(self, a): + """ + return: + """ + img = self.Image.open(BytesIO(base64.b64decode(a))) + if img.mode != "RGB": + img = img.convert("RGB") + img = np.array(img) + return img + + + +if __name__ == "__main__": + LOCK = Lock() + AVAIL_DEVICE_LIST = config.AVAIL_DEVICE_LIST + AVAIL_DEVICE_MEMFRAC = config.AVAIL_DEVICE_MEMFRAC + AVAIL_DEVICE_MAXTHREAD = config.AVAIL_DEVICE_MAXTHREAD + + proc_list = [] + print('{} GPUs Available'.format(len(AVAIL_DEVICE_LIST))) + if AVAIL_DEVICE_LIST: + for index, device in enumerate(AVAIL_DEVICE_LIST): + thread_count = int(AVAIL_DEVICE_MAXTHREAD[index]) + mem_frac = float(AVAIL_DEVICE_MEMFRAC[index]) + if config.MAX_FRAC < mem_frac: + mem_frac = config.MAX_FRAC + print('Preparing {} process on GPU: {}, frac: {}'.format(thread_count, device.id, mem_frac)) + if config.MAX_THREADS < thread_count: + thread_count = config.MAX_THREADS + for thread in range(thread_count): + p = mlWorker(LOCK, GPU=device, FRAC=mem_frac) + p.daemon = True + proc_list.append(p) + print('Starting total: {} processes'.format(len(proc_list))) + for proc in proc_list: + proc.start() + print('All processes started') + else: + p = mlWorker(LOCK) + p.daemon = True + p.start() + p.join() + + if proc_list: + for proc in proc_list: + proc.join() diff --git a/config.py b/config.py new file mode 100644 index 000000000..c6fb41df9 --- /dev/null +++ b/config.py @@ -0,0 +1,83 @@ +import os +import urllib.request +import shutil +from mrcnn.tools.config import Config + + +ALLOWED_EXTENSIONS = set(['jpg', 'jpeg']) + +ROOT_DIR = os.getcwd() +UPLOAD_FOLDER = os.path.join(ROOT_DIR, "images") +if not os.path.exists(UPLOAD_FOLDER): + os.makedirs(UPLOAD_FOLDER) +MODEL_DIR = os.path.join(ROOT_DIR, "logs") +if not os.path.exists(MODEL_DIR): + os.makedirs(MODEL_DIR) +# Local path to trained weights file +MODEL_DIR = os.path.join(ROOT_DIR, "weights") +if not os.path.exists(COCO_MODEL_DIR): + os.makedirs(COCO_MODEL_DIR) +MODEL_PATH = os.path.join(ROOT_DIR, "weights/mask_rcnn_coco.h5") + +def download_trained_weights(coco_model_path, verbose=1): + """Download COCO trained weights from Releases. + + coco_model_path: local path of COCO trained weights + """ + MODEL_URL = "https://pjreddie.com/media/files/yolov3.weights" + if verbose > 0: + print("Downloading pretrained model to " + MODEL_PATH + " ...") + with urllib.request.urlopen(MODEL_URL) as resp, open(MODEL_PATH, 'wb') as out: + shutil.copyfileobj(resp, out) + if verbose > 0: + print("... done downloading pretrained model!") + +# Download COCO trained weights from Releases if needed +if not os.path.exists(MODEL_PATH): + download_trained_weights(MODEL_PATH, verbose=VERBOSE) + +import GPUtil + +LEAST_GMEM = 2250 # MB +MAX_THREADS = 1 +MIN_FRAC = 0.3 +MAX_FRAC = 0.3 +GPU_LAOD = 0.5 +GMEM_LAOD_LIMIT = 1.0 +AVAIL_DEVICE_LIST = [] +AVAIL_DEVICE_MAT = [] +AVAIL_DEVICE_MEMFRAC = [] +AVAIL_DEVICE_MAXTHREAD = [] +try: + GPUs = GPUtil.getGPUs() + Gall = '' + Gfree = '' + for GPU in GPUs: + Gall = GPU.memoryTotal + Gfree = GPU.memoryFree + GMEM_LAOD_LIMIT = float(format(float(LEAST_GMEM / Gall), '.2f')) + if int(GPUtil.getAvailability([GPU], maxLoad=GPU_LAOD, maxMemory=GMEM_LAOD_LIMIT)) == 1: + AVAIL_DEVICE_LIST.append(GPU) + if GMEM_LAOD_LIMIT < MIN_FRAC: + GMEM_LAOD_LIMIT = MIN_FRAC + if GMEM_LAOD_LIMIT > MAX_FRAC: + GMEM_LAOD_LIMIT = MAX_FRAC + AVAIL_DEVICE_MEMFRAC.append(GMEM_LAOD_LIMIT) + AVAIL_DEVICE_MAXTHREAD.append(int(1.0/GMEM_LAOD_LIMIT)) +except Exception as e: + print(e) + +# initialize Redis connection settings +REDIS_HOST = "localhost" +REDIS_PORT = 6379 +REDIS_DB = 0 + +BATCH_SIZE = 1 +# initialize constants used for server queuing +IMAGE_QUEUE = "yolo3_queue" + +SERVER_SLEEP = 0.1 +CLIENT_SLEEP = 0.1 + +# Output Throttle +THROTTLE = 0.9 diff --git a/yolo3_cam.py b/yolo3_cam.py index 47b86c073..cd6a9567f 100644 --- a/yolo3_cam.py +++ b/yolo3_cam.py @@ -16,7 +16,7 @@ # set some parameters net_h, net_w = 416, 416 -obj_thresh, nms_thresh = 0.5, 0.45 +obj_thresh, nms_thresh = 0.7, 0.7 anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ @@ -62,7 +62,7 @@ def _main_(args): cap.set(4, 720) while(True): # Capture frame-by-frame - ret, image = cap.read() + _, image = cap.read() # preprocess the image new_image = preprocess_input(image, net_h, net_w) # run the prediction @@ -70,7 +70,6 @@ def _main_(args): boxes = [] frame = post_stream(yolos, boxes, image) cv2.imshow('Yolo3', frame) - if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() diff --git a/yolo3_cam_mp.py b/yolo3_cam_mp.py new file mode 100644 index 000000000..97745bc74 --- /dev/null +++ b/yolo3_cam_mp.py @@ -0,0 +1,106 @@ +import argparse +import os + +import struct +import cv2 +import numpy as np +import multiprocessing +from multiprocessing import Process, Queue +from utils.weightreader import WeightReader +from utils.bbox import BoundBox +from utils.tools import preprocess_input, decode_netout +from utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from model.yolo3 import make_yolov3_model + +np.set_printoptions(threshold=np.nan) +os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"]="0" + +taskqueue = Queue() +resqueue = Queue() + +# set some parameters +net_h, net_w = 416, 416 +obj_thresh, nms_thresh = 0.7, 0.7 +anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] +labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ + "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ + "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ + "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ + "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ + "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] + +def post_stream(yolos, boxes, image): + image_h, image_w, _ = image.shape + for i in range(len(yolos)): + # decode the output of the network + boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) + + # correct the sizes of the bounding boxes + correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) + # suppress non-maximal boxes + do_nms(boxes, nms_thresh) + # draw bounding boxes on the image using labels + i = draw_boxes(image, boxes, labels, obj_thresh) + return i + +def detect_loop(model, taskqueue): + while True: + image = taskqueue.get() + if image is None: + continue + res = model.predict(image) + boxes = [] + frame = post_stream(res, boxes, image) + resqueue.put(frame) + +def image_display(resqueue): + while True: + if resqueue.empty(): + continue + else: + image = resqueue.get() + cv2.imshow ('image_display', image) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + +def _main_(args): + weights_path = args.weights + + # make the yolov3 model to predict 80 classes on COCO + yolov3 = make_yolov3_model() + + # load the weights trained on COCO into the model + weight_reader = WeightReader(weights_path) + weight_reader.load_weights(yolov3) + + cap = cv2.VideoCapture(0) + cap.set(3, 1280) # set the Horizontal resolution + cap.set(4, 720) + p = Process(target=detect_loop, args=(yolov3, taskqueue,)) + p.start() + q = Process(target=image_display, args=(resqueue,)) + q.start() + while(True): + # Capture frame-by-frame + _, image = cap.read() + # preprocess the image + image = preprocess_input(image, net_h, net_w) + taskqueue.put(image) + cap.release() + cv2.destroyAllWindows() + +if __name__ == '__main__': + argparser = argparse.ArgumentParser( + description='test yolov3 network with coco weights') + argparser.add_argument( + '-w', + '--weights', + help='path to weights file') + + args = argparser.parse_args() + _main_(args) From fc653f47e37268625293fe4a750aeee45ca6bb5e Mon Sep 17 00:00:00 2001 From: kcw Date: Wed, 6 Jun 2018 16:02:51 +0800 Subject: [PATCH 13/15] _ --- RMS_yolo3.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/RMS_yolo3.py b/RMS_yolo3.py index ba92303d8..0ac94c613 100644 --- a/RMS_yolo3.py +++ b/RMS_yolo3.py @@ -139,7 +139,7 @@ def mainloop(self, model='', graph=''): # loop over the image IDs and their corresponding set of # results from our model output = [] - output = self.extract_result(results, + output = self.extract_result(results, labels, throttle=float(thresholds[imageID])) redisdbSession.set(imageID, json.dumps(output)) # sleep for a small amount @@ -149,17 +149,30 @@ def mainloop(self, model='', graph=''): time.sleep(config.SERVER_SLEEP) continue - def extract_result(self, results, throttle='0.95'): + def extract_result(self, results, labels, throttle='0.95'): boxes = [] for i in range(len(yolos)): # decode the output of the network - boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) + boxes.append(decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w)) # correct the sizes of the bounding boxes correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) # suppress non-maximal boxes - do_nms(boxes, nms_thresh) - + do_nms(boxes, nms_thresh) + output = [] + for box in boxes: + label_str = '' + label = -1 + for i in range(len(labels)): + if box.classes[i] > obj_thresh: + label_str += labels[i] + label = i + if label >= 0: + output.append({ + bbox: [box.xmin,box.ymin,box.xmax,box.ymax], + label: label_str + score: str(box.get_score()) + }) return output def base64_decode_image(self, a): From dbdd980f298b32e5c06183d39a7157ac5fe77313 Mon Sep 17 00:00:00 2001 From: kcw Date: Mon, 11 Jun 2018 12:06:53 +0800 Subject: [PATCH 14/15] re-infra, add openimg code --- RMS_yolo3.py => RMS/RMS_yolo3.py | 0 config.py => RMS/config.py | 0 anchors/gen_openimg_anchors.py | 133 ++++++++++++++++++ gen_anchors.py => anchors/gen_voc_anchors.py | 0 dataset/__init__.py | 0 dataset/csv2xml.py | 104 ++++++++++++++ dataset/openimg.py | 84 +++++++++++ voc.py => dataset/voc.py | 0 yolo3_cam.py => demo/yolo3_cam.py | 10 +- yolo3_cam_mp.py => demo/yolo3_cam_mp.py | 10 +- .../yolo3_one_file_to_detect_them_all.py | 10 +- yolo.py => model/yolo.py | 0 train.py | 8 +- 13 files changed, 340 insertions(+), 19 deletions(-) rename RMS_yolo3.py => RMS/RMS_yolo3.py (100%) rename config.py => RMS/config.py (100%) create mode 100644 anchors/gen_openimg_anchors.py rename gen_anchors.py => anchors/gen_voc_anchors.py (100%) create mode 100644 dataset/__init__.py create mode 100644 dataset/csv2xml.py create mode 100644 dataset/openimg.py rename voc.py => dataset/voc.py (100%) rename yolo3_cam.py => demo/yolo3_cam.py (92%) rename yolo3_cam_mp.py => demo/yolo3_cam_mp.py (93%) rename yolo3_one_file_to_detect_them_all.py => demo/yolo3_one_file_to_detect_them_all.py (92%) rename yolo.py => model/yolo.py (100%) diff --git a/RMS_yolo3.py b/RMS/RMS_yolo3.py similarity index 100% rename from RMS_yolo3.py rename to RMS/RMS_yolo3.py diff --git a/config.py b/RMS/config.py similarity index 100% rename from config.py rename to RMS/config.py diff --git a/anchors/gen_openimg_anchors.py b/anchors/gen_openimg_anchors.py new file mode 100644 index 000000000..9babc9df9 --- /dev/null +++ b/anchors/gen_openimg_anchors.py @@ -0,0 +1,133 @@ +import random +import argparse +import numpy as np + +from ..dataset.openimg import parse_openimg_annotation + +import json + +def IOU(ann, centroids): + w, h = ann + similarities = [] + + for centroid in centroids: + c_w, c_h = centroid + + if c_w >= w and c_h >= h: + similarity = w*h/(c_w*c_h) + elif c_w >= w and c_h <= h: + similarity = w*c_h/(w*h + (c_w-w)*c_h) + elif c_w <= w and c_h >= h: + similarity = c_w*h/(w*h + c_w*(c_h-h)) + else: #means both w,h are bigger than c_w and c_h respectively + similarity = (c_w*c_h)/(w*h) + similarities.append(similarity) # will become (k,) shape + + return np.array(similarities) + +def avg_IOU(anns, centroids): + n,d = anns.shape + sum = 0. + + for i in range(anns.shape[0]): + sum+= max(IOU(anns[i], centroids)) + + return sum/n + +def print_anchors(centroids): + out_string = '' + + anchors = centroids.copy() + + widths = anchors[:, 0] + sorted_indices = np.argsort(widths) + + r = "anchors: [" + for i in sorted_indices: + out_string += str(int(anchors[i,0]*416)) + ',' + str(int(anchors[i,1]*416)) + ', ' + + print(out_string[:-2]) + +def run_kmeans(ann_dims, anchor_num): + ann_num = ann_dims.shape[0] + iterations = 0 + prev_assignments = np.ones(ann_num)*(-1) + iteration = 0 + old_distances = np.zeros((ann_num, anchor_num)) + + indices = [random.randrange(ann_dims.shape[0]) for i in range(anchor_num)] + centroids = ann_dims[indices] + anchor_dim = ann_dims.shape[1] + + while True: + distances = [] + iteration += 1 + for i in range(ann_num): + d = 1 - IOU(ann_dims[i], centroids) + distances.append(d) + distances = np.array(distances) # distances.shape = (ann_num, anchor_num) + + print("iteration {}: dists = {}".format(iteration, np.sum(np.abs(old_distances-distances)))) + + #assign samples to centroids + assignments = np.argmin(distances,axis=1) + + if (assignments == prev_assignments).all() : + return centroids + + #calculate new centroids + centroid_sums=np.zeros((anchor_num, anchor_dim), np.float) + for i in range(ann_num): + centroid_sums[assignments[i]]+=ann_dims[i] + for j in range(anchor_num): + centroids[j] = centroid_sums[j]/(np.sum(assignments==j) + 1e-6) + + prev_assignments = assignments.copy() + old_distances = distances.copy() + +def _main_(argv): + config_path = args.conf + num_anchors = args.anchors + + with open(config_path) as config_buffer: + config = json.loads(config_buffer.read()) + + train_imgs, train_labels = parse_openimg_annotation( + config['train']['train_annot_folder'], + config['train']['train_image_folder'], + config['train']['cache_name'], + config['model']['labels'] + ) + + # run k_mean to find the anchors + annotation_dims = [] + for image in train_imgs: + print(image['filename']) + for obj in image['object']: + relative_w = (float(obj['xmax']) - float(obj['xmin']))/image['width'] + relatice_h = (float(obj["ymax"]) - float(obj['ymin']))/image['height'] + annotation_dims.append(tuple(map(float, (relative_w,relatice_h)))) + + annotation_dims = np.array(annotation_dims) + centroids = run_kmeans(annotation_dims, num_anchors) + + # write anchors to file + print('\naverage IOU for', num_anchors, 'anchors:', '%0.2f' % avg_IOU(annotation_dims, centroids)) + print_anchors(centroids) + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + + argparser.add_argument( + '-c', + '--conf', + default='config.json', + help='path to configuration file') + argparser.add_argument( + '-a', + '--anchors', + default=9, + help='number of anchors to use') + + args = argparser.parse_args() + _main_(args) diff --git a/gen_anchors.py b/anchors/gen_voc_anchors.py similarity index 100% rename from gen_anchors.py rename to anchors/gen_voc_anchors.py diff --git a/dataset/__init__.py b/dataset/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dataset/csv2xml.py b/dataset/csv2xml.py new file mode 100644 index 000000000..91fb0dc0e --- /dev/null +++ b/dataset/csv2xml.py @@ -0,0 +1,104 @@ +''' +### CSV Format ### +ImageID,Source,LabelName,Confidence,XMin,XMax,YMin,YMax,IsOccluded,IsTruncated,IsGroupOf,IsDepiction,IsInside +000026e7ee790996,freeform,/m/07j7r,1,0.071905,0.145346,0.206591,0.391306,0,1,1,0,0 +### File Format ### +/data/ + - /images/ + - /train/ + - /validation/ + - /test/ + - /annotations/ + - train-annotations-bbox.csv + - validation-annotations-bbox.csv + - test-annotations-bbox.csv + - class-descriptions-boxable.csv +''' +import numpy as np +import os +import pickle +import pandas +from lxml.etree import Element, SubElement, tostring +from xml.dom.minidom import parseString +from PIL import Image + +def parse_tfrecord_annotation(ann_file, img_dir, lablefile, cache_name, labels=[]): + ''' + ann_file: /data/annotations/train-annotations-bbox.csv + img_dir: /data/images/train/ + ''' + if os.path.exists(cache_name): + with open(cache_name, 'rb') as handle: + cache = pickle.load(handle) + all_insts, seen_labels = cache['all_insts'], cache['seen_labels'] + else: + all_insts = [] + seen_labels = {} + + try: + csv = pandas.read_csv(ann_file).values + except Exception as e: + print(e) + print('Ignore this bad annotation: ' + ann_file) + continue + img = {} + for row in csv: + if not row[0] in img: + fn = row[0] + '.jpg' + img[row[0]] = { + 'filename': fn, + 'path': img_dir + fn + } + im = Image.open(img[row[0]]['path']) + img[row[0]]['size'] = {} + img[row[0]]['size']['width'], img[row[0]]['size']['height'] = im.size + img[row[0]]['size']['depth'] = 3 + img[row[0]]['object'] = [] + img[row[0]]['object'].append({ + 'name': row[2] + }) + + + for elem in tree.iter(): + if 'filename' in elem.tag: + img['filename'] = img_dir + elem.text + if 'width' in elem.tag: + img['width'] = int(elem.text) + if 'height' in elem.tag: + img['height'] = int(elem.text) + if 'object' in elem.tag or 'part' in elem.tag: + obj = {} + + for attr in list(elem): + if 'name' in attr.tag: + obj['name'] = attr.text + + if obj['name'] in seen_labels: + seen_labels[obj['name']] += 1 + else: + seen_labels[obj['name']] = 1 + + if len(labels) > 0 and obj['name'] not in labels: + break + else: + img['object'] += [obj] + + if 'bndbox' in attr.tag: + for dim in list(attr): + if 'xmin' in dim.tag: + obj['xmin'] = int(round(float(dim.text))) + if 'ymin' in dim.tag: + obj['ymin'] = int(round(float(dim.text))) + if 'xmax' in dim.tag: + obj['xmax'] = int(round(float(dim.text))) + if 'ymax' in dim.tag: + obj['ymax'] = int(round(float(dim.text))) + + if len(img['object']) > 0: + all_insts += [img] + + cache = {'all_insts': all_insts, 'seen_labels': seen_labels} + with open(cache_name, 'wb') as handle: + pickle.dump(cache, handle, protocol=pickle.HIGHEST_PROTOCOL) + + return all_insts, seen_labels \ No newline at end of file diff --git a/dataset/openimg.py b/dataset/openimg.py new file mode 100644 index 000000000..5bfb27780 --- /dev/null +++ b/dataset/openimg.py @@ -0,0 +1,84 @@ +''' +### CSV Format ### +ImageID,Source,LabelName,Confidence,XMin,XMax,YMin,YMax,IsOccluded,IsTruncated,IsGroupOf,IsDepiction,IsInside +000026e7ee790996,freeform,/m/07j7r,1,0.071905,0.145346,0.206591,0.391306,0,1,1,0,0 +### File Format ### +/data/ + - /images/ + - /train/ + - /validation/ + - /test/ + - /annotations/ + - train-annotations-bbox.csv + - validation-annotations-bbox.csv + - test-annotations-bbox.csv + - class-descriptions-boxable.csv +''' +import numpy as np +import os +import xml.etree.ElementTree as ET +import pickle +import numpy as np +import os +import pickle +import pandas +from lxml.etree import Element, SubElement, tostring +from xml.dom.minidom import parseString +from PIL import Image + +def parse_openimg_annotation(ann_file, img_dir, lable_map, cache_name, labels=[]): + if os.path.exists(cache_name): + with open(cache_name, 'rb') as handle: + cache = pickle.load(handle) + all_insts, seen_labels = cache['all_insts'], cache['seen_labels'] + else: + all_insts = [] + seen_labels = {} + imgs = {} + label_map = {} + + try: + img_csv = pandas.read_csv(ann_file).values + label_csv = pandas.read_csv(lable_map).values + except Exception as e: + print(e) + print('Ignore this bad annotation: ' + ann_dir + ann) + continue + + for row in label_csv: + label_map[row[0]] = row[1] + + for row in img_csv: + if not row[0] in imgs: + imgs[row[0]] = {} + imgs[row[0]]['filename'] = os.path.join(img_dir, row[0] + '.jpg') + im = Image.open(imgs[row[0]]['filename']) + imgs[row[0]]['width'], imgs[row[0]]['height'] = im.size + imgs[row[0]]['object'] = [] + + label_id = row[2] + label_name = label_map[label_id] + if label_name in seen_labels: + seen_labels[label_name] += 1 + else: + seen_labels[label_name] = 1 + if len(labels) > 0 and label_name not in labels: + break + else: + obj = { + 'name': label_name, + 'xmin': int(round(float(row[4] * img['width']))), + 'ymin': int(round(float(row[5] * img['height']))), + 'xmax': int(round(float(row[6] * img['width']))), + 'ymax': int(round(float(row[7] * img['height'])))} + imgs[row[0]]['object'].append(obj) + + for key, img in imgs: + if len(img['object']) > 0: + all_insts += [img] + + cache = {'all_insts': all_insts, 'seen_labels': seen_labels} + with open(cache_name, 'wb') as handle: + pickle.dump(cache, handle, protocol=pickle.HIGHEST_PROTOCOL) + + return all_insts, seen_labels \ No newline at end of file diff --git a/voc.py b/dataset/voc.py similarity index 100% rename from voc.py rename to dataset/voc.py diff --git a/yolo3_cam.py b/demo/yolo3_cam.py similarity index 92% rename from yolo3_cam.py rename to demo/yolo3_cam.py index cd6a9567f..04086e7e7 100644 --- a/yolo3_cam.py +++ b/demo/yolo3_cam.py @@ -4,11 +4,11 @@ import struct import cv2 import numpy as np -from utils.weightreader import WeightReader -from utils.bbox import BoundBox -from utils.tools import preprocess_input, decode_netout -from utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from model.yolo3 import make_yolov3_model +from ..utils.weightreader import WeightReader +from ..utils.bbox import BoundBox +from ..utils.tools import preprocess_input, decode_netout +from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from ..model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" diff --git a/yolo3_cam_mp.py b/demo/yolo3_cam_mp.py similarity index 93% rename from yolo3_cam_mp.py rename to demo/yolo3_cam_mp.py index 97745bc74..ec66def38 100644 --- a/yolo3_cam_mp.py +++ b/demo/yolo3_cam_mp.py @@ -6,11 +6,11 @@ import numpy as np import multiprocessing from multiprocessing import Process, Queue -from utils.weightreader import WeightReader -from utils.bbox import BoundBox -from utils.tools import preprocess_input, decode_netout -from utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from model.yolo3 import make_yolov3_model +from ..utils.weightreader import WeightReader +from ..utils.bbox import BoundBox +from ..utils.tools import preprocess_input, decode_netout +from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from ..model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" diff --git a/yolo3_one_file_to_detect_them_all.py b/demo/yolo3_one_file_to_detect_them_all.py similarity index 92% rename from yolo3_one_file_to_detect_them_all.py rename to demo/yolo3_one_file_to_detect_them_all.py index 248c78b5b..c0c482bbd 100644 --- a/yolo3_one_file_to_detect_them_all.py +++ b/demo/yolo3_one_file_to_detect_them_all.py @@ -6,11 +6,11 @@ from keras.models import Model import struct import cv2 -from utils.weightreader import WeightReader -from utils.bbox import BoundBox -from utils.tools import preprocess_input, decode_netout -from utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from model.yolo3 import make_yolov3_model +from ..utils.weightreader import WeightReader +from ..utils.bbox import BoundBox +from ..utils.tools import preprocess_input, decode_netout +from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from ..model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" diff --git a/yolo.py b/model/yolo.py similarity index 100% rename from yolo.py rename to model/yolo.py diff --git a/train.py b/train.py index 491289da9..85f9fcdd2 100644 --- a/train.py +++ b/train.py @@ -4,14 +4,14 @@ import os import numpy as np import json -from voc import parse_voc_annotation -from yolo import create_yolov3_model, dummy_loss +from .dataset.voc import parse_voc_annotation +from .model.yolo import create_yolov3_model, dummy_loss from generator import BatchGenerator -from utils.utils import normalize, evaluate, makedirs +from .utils.utils import normalize, evaluate, makedirs from keras.callbacks import EarlyStopping, ReduceLROnPlateau from keras.optimizers import Adam from callbacks import CustomModelCheckpoint, CustomTensorBoard -from utils.multi_gpu_model import multi_gpu_model +from .utils.multi_gpu_model import multi_gpu_model import tensorflow as tf import keras from keras.models import load_model From 24ce74fcf5cf1d7ef3adb0e33283559de1a7b245 Mon Sep 17 00:00:00 2001 From: clement10601 Date: Tue, 12 Jun 2018 14:14:26 +0800 Subject: [PATCH 15/15] code modified --- anchors/gen_openimg_anchors.py | 14 +- config/config_carplate.json | 38 +++ config/config_openimg.json | 41 ++++ config.json => config/config_sample.json | 6 +- dataset/openimg.py | 50 ++-- demo/yolo3_cam.py | 13 +- demo/yolo3_cam_mp.py | 106 -------- demo/yolo3_one_file_to_detect_them_all.py | 13 +- train_openimg.py | 282 ++++++++++++++++++++++ 9 files changed, 417 insertions(+), 146 deletions(-) create mode 100644 config/config_carplate.json create mode 100644 config/config_openimg.json rename config.json => config/config_sample.json (84%) delete mode 100644 demo/yolo3_cam_mp.py create mode 100644 train_openimg.py diff --git a/anchors/gen_openimg_anchors.py b/anchors/gen_openimg_anchors.py index 9babc9df9..8c45d3005 100644 --- a/anchors/gen_openimg_anchors.py +++ b/anchors/gen_openimg_anchors.py @@ -1,11 +1,13 @@ +import os, sys import random import argparse import numpy as np - -from ..dataset.openimg import parse_openimg_annotation - import json +ROOT_DIR = os.path.abspath("../") +sys.path.append(ROOT_DIR) +from dataset.openimg import parse_openimg_annotation + def IOU(ann, centroids): w, h = ann similarities = [] @@ -93,8 +95,9 @@ def _main_(argv): config = json.loads(config_buffer.read()) train_imgs, train_labels = parse_openimg_annotation( - config['train']['train_annot_folder'], + config['train']['train_annot_file'], config['train']['train_image_folder'], + config['train']['label_map'], config['train']['cache_name'], config['model']['labels'] ) @@ -103,11 +106,12 @@ def _main_(argv): annotation_dims = [] for image in train_imgs: print(image['filename']) + print(image) for obj in image['object']: relative_w = (float(obj['xmax']) - float(obj['xmin']))/image['width'] relatice_h = (float(obj["ymax"]) - float(obj['ymin']))/image['height'] annotation_dims.append(tuple(map(float, (relative_w,relatice_h)))) - + print(train_imgs) annotation_dims = np.array(annotation_dims) centroids = run_kmeans(annotation_dims, num_anchors) diff --git a/config/config_carplate.json b/config/config_carplate.json new file mode 100644 index 000000000..0da74d04e --- /dev/null +++ b/config/config_carplate.json @@ -0,0 +1,38 @@ +{ + "model" : { + "min_input_size": 352, + "max_input_size": 448, + "anchors": [2,5, 2,7, 3,8, 4,11, 5,15, 7,20, 11,27, 16,46, 29,59], + "labels": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9","A", "B", "C", "D", "E", "F", + "G", "H", "I", "J", "K", "L", "M","N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "carPlate"] + }, + + "train": { + "train_image_folder": "/workspace/carPlateDataset/20180607/JPEGImages/", + "train_annot_folder": "/workspace/carPlateDataset/20180607/Annotations/", + "cache_name": "cache/carplate_train.pkl", + + "train_times": 8, + "batch_size": 12, + "learning_rate": 1e-4, + "nb_epochs": 100, + "warmup_epochs": 3, + "ignore_thresh": 0.5, + "gpus": "0", + "grid_scales": [1,1,1], + "obj_scale": 5, + "noobj_scale": 1, + "xywh_scale": 1, + "class_scale": 1, + "tensorboard_dir": "logs", + "saved_weights_name": "weights/carplate.h5", + "debug": false + }, + "valid": { + "valid_image_folder": "", + "valid_annot_folder": "", + "cache_name": "", + + "valid_times": 1 + } +} diff --git a/config/config_openimg.json b/config/config_openimg.json new file mode 100644 index 000000000..6730746d3 --- /dev/null +++ b/config/config_openimg.json @@ -0,0 +1,41 @@ +{ + "model" : { + "min_input_size": 448, + "max_input_size": 448, + "anchors": [55,69, 75,234, 133,240, 136,129, 142,363, 203,290, 228,184, 285,359, 341,260], + "labels": ["Person"] + }, + + "train": { + "train_image_folder": "/data1/openimages/600c_bbox/images/train/", + "train_annot_file": "/data1/openimages/600c_bbox/annotation/train-annotations-bbox.csv", + "train_annot_folder": "/data1/openimages/600c_bbox/annotation/", + "cache_name": "cache/openimg_train.pkl", + "label_map": "/data1/openimages/600c_bbox/annotation/class-descriptions-boxable.csv", + "train_times": 8, + "batch_size": 16, + "learning_rate": 1e-4, + "nb_epochs": 100, + "warmup_epochs": 3, + "ignore_thresh": 0.5, + "gpus": "0,1", + + "grid_scales": [1,1,1], + "obj_scale": 5, + "noobj_scale": 1, + "xywh_scale": 1, + "class_scale": 1, + + "tensorboard_dir": "logs", + "saved_weights_name": "weights/openimg_600c.h5", + "debug": true + }, + + "valid": { + "valid_image_folder": "/data1/openimages/600c_bbox/images/validation/", + "valid_annot_folder": "/data1/openimages/600c_bbox/annotation/", + "train_annot_file": "validation-annotations-bbox.csv", + "cache_name": "openimg_val.pkl", + "valid_times": 1 + } +} diff --git a/config.json b/config/config_sample.json similarity index 84% rename from config.json rename to config/config_sample.json index 31722ac92..b2830a32a 100644 --- a/config.json +++ b/config/config_sample.json @@ -7,9 +7,9 @@ }, "train": { - "train_image_folder": "/home/andy/Desktop/github/kangaroo/images/", + "train_image_folder": "/data1/openimages/600c_bbox/images/train/", "train_annot_folder": "/home/andy/Desktop/github/kangaroo/annots/", - "cache_name": "kangaroo_train.pkl", + "cache_name": cache/"kangaroo_train.pkl", "train_times": 8, "batch_size": 16, @@ -26,7 +26,7 @@ "class_scale": 1, "tensorboard_dir": "logs", - "saved_weights_name": "kangaroo.h5", + "saved_weights_name": "weights/kangaroo.h5", "debug": true }, diff --git a/dataset/openimg.py b/dataset/openimg.py index 5bfb27780..bc60356a7 100644 --- a/dataset/openimg.py +++ b/dataset/openimg.py @@ -19,12 +19,13 @@ import xml.etree.ElementTree as ET import pickle import numpy as np -import os +import os, sys import pickle import pandas from lxml.etree import Element, SubElement, tostring from xml.dom.minidom import parseString from PIL import Image +import glob def parse_openimg_annotation(ann_file, img_dir, lable_map, cache_name, labels=[]): if os.path.exists(cache_name): @@ -38,42 +39,47 @@ def parse_openimg_annotation(ann_file, img_dir, lable_map, cache_name, labels=[] label_map = {} try: - img_csv = pandas.read_csv(ann_file).values - label_csv = pandas.read_csv(lable_map).values + img_csv = pandas.read_csv(ann_file, sep=',', header = None, skiprows=1, chunksize=1, dtype=str) + label_csv = pandas.read_csv(lable_map,sep=',', header = None, chunksize=1) except Exception as e: print(e) print('Ignore this bad annotation: ' + ann_dir + ann) - continue - for row in label_csv: - label_map[row[0]] = row[1] + label_map[str(row[0].iloc[0])] = str(row[1].iloc[0]) for row in img_csv: - if not row[0] in imgs: - imgs[row[0]] = {} - imgs[row[0]]['filename'] = os.path.join(img_dir, row[0] + '.jpg') - im = Image.open(imgs[row[0]]['filename']) - imgs[row[0]]['width'], imgs[row[0]]['height'] = im.size - imgs[row[0]]['object'] = [] + iid = str(row[0].iloc[0]) + if not iid in imgs: + imgs[iid] = {} + imgs[iid]['object'] = [] + imgs[iid]['filename'] = os.path.join(img_dir, iid + '.jpg') + try: + im = Image.open(imgs[iid]['filename']) + imgs[iid]['width'], imgs[iid]['height'] = im.size + except: + npath = glob.glob(os.path.join(img_dir, iid) + '.*') + imgs[iid]['filename'] = npath[0] + im = Image.open(imgs[iid]['filename']) + imgs[iid]['width'], imgs[iid]['height'] = im.size - label_id = row[2] + label_id = str(row[2].iloc[0]) label_name = label_map[label_id] if label_name in seen_labels: seen_labels[label_name] += 1 else: seen_labels[label_name] = 1 if len(labels) > 0 and label_name not in labels: - break + continue else: obj = { 'name': label_name, - 'xmin': int(round(float(row[4] * img['width']))), - 'ymin': int(round(float(row[5] * img['height']))), - 'xmax': int(round(float(row[6] * img['width']))), - 'ymax': int(round(float(row[7] * img['height'])))} - imgs[row[0]]['object'].append(obj) - - for key, img in imgs: + 'xmin': int(round(float(row[4].iloc[0]) * imgs[iid]['width'])), + 'ymin': int(round(float(row[5].iloc[0]) * imgs[iid]['height'])), + 'xmax': int(round(float(row[6].iloc[0]) * imgs[iid]['width'])), + 'ymax': int(round(float(row[7].iloc[0]) * imgs[iid]['height']))} + imgs[iid]['object'].append(obj) + print(imgs) + for key, img in imgs.items(): if len(img['object']) > 0: all_insts += [img] @@ -81,4 +87,4 @@ def parse_openimg_annotation(ann_file, img_dir, lable_map, cache_name, labels=[] with open(cache_name, 'wb') as handle: pickle.dump(cache, handle, protocol=pickle.HIGHEST_PROTOCOL) - return all_insts, seen_labels \ No newline at end of file + return all_insts, seen_labels diff --git a/demo/yolo3_cam.py b/demo/yolo3_cam.py index 04086e7e7..04f2e17cc 100644 --- a/demo/yolo3_cam.py +++ b/demo/yolo3_cam.py @@ -4,11 +4,14 @@ import struct import cv2 import numpy as np -from ..utils.weightreader import WeightReader -from ..utils.bbox import BoundBox -from ..utils.tools import preprocess_input, decode_netout -from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from ..model.yolo3 import make_yolov3_model +import sys +ROOT_DIR = os.path.abspath("../") +sys.path.append(ROOT_DIR) +from utils.weightreader import WeightReader +from utils.bbox import BoundBox +from utils.tools import preprocess_input, decode_netout +from utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" diff --git a/demo/yolo3_cam_mp.py b/demo/yolo3_cam_mp.py deleted file mode 100644 index ec66def38..000000000 --- a/demo/yolo3_cam_mp.py +++ /dev/null @@ -1,106 +0,0 @@ -import argparse -import os - -import struct -import cv2 -import numpy as np -import multiprocessing -from multiprocessing import Process, Queue -from ..utils.weightreader import WeightReader -from ..utils.bbox import BoundBox -from ..utils.tools import preprocess_input, decode_netout -from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from ..model.yolo3 import make_yolov3_model - -np.set_printoptions(threshold=np.nan) -os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" -os.environ["CUDA_VISIBLE_DEVICES"]="0" - -taskqueue = Queue() -resqueue = Queue() - -# set some parameters -net_h, net_w = 416, 416 -obj_thresh, nms_thresh = 0.7, 0.7 -anchors = [[116,90, 156,198, 373,326], [30,61, 62,45, 59,119], [10,13, 16,30, 33,23]] -labels = ["person", "bicycle", "car", "motorbike", "aeroplane", "bus", "train", "truck", \ - "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", \ - "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", \ - "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", \ - "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", \ - "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", \ - "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", \ - "chair", "sofa", "pottedplant", "bed", "diningtable", "toilet", "tvmonitor", "laptop", "mouse", \ - "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", \ - "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush"] - -def post_stream(yolos, boxes, image): - image_h, image_w, _ = image.shape - for i in range(len(yolos)): - # decode the output of the network - boxes += decode_netout(yolos[i][0], anchors[i], obj_thresh, net_h, net_w) - - # correct the sizes of the bounding boxes - correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w) - # suppress non-maximal boxes - do_nms(boxes, nms_thresh) - # draw bounding boxes on the image using labels - i = draw_boxes(image, boxes, labels, obj_thresh) - return i - -def detect_loop(model, taskqueue): - while True: - image = taskqueue.get() - if image is None: - continue - res = model.predict(image) - boxes = [] - frame = post_stream(res, boxes, image) - resqueue.put(frame) - -def image_display(resqueue): - while True: - if resqueue.empty(): - continue - else: - image = resqueue.get() - cv2.imshow ('image_display', image) - if cv2.waitKey(1) & 0xFF == ord('q'): - break - -def _main_(args): - weights_path = args.weights - - # make the yolov3 model to predict 80 classes on COCO - yolov3 = make_yolov3_model() - - # load the weights trained on COCO into the model - weight_reader = WeightReader(weights_path) - weight_reader.load_weights(yolov3) - - cap = cv2.VideoCapture(0) - cap.set(3, 1280) # set the Horizontal resolution - cap.set(4, 720) - p = Process(target=detect_loop, args=(yolov3, taskqueue,)) - p.start() - q = Process(target=image_display, args=(resqueue,)) - q.start() - while(True): - # Capture frame-by-frame - _, image = cap.read() - # preprocess the image - image = preprocess_input(image, net_h, net_w) - taskqueue.put(image) - cap.release() - cv2.destroyAllWindows() - -if __name__ == '__main__': - argparser = argparse.ArgumentParser( - description='test yolov3 network with coco weights') - argparser.add_argument( - '-w', - '--weights', - help='path to weights file') - - args = argparser.parse_args() - _main_(args) diff --git a/demo/yolo3_one_file_to_detect_them_all.py b/demo/yolo3_one_file_to_detect_them_all.py index c0c482bbd..fee5610db 100644 --- a/demo/yolo3_one_file_to_detect_them_all.py +++ b/demo/yolo3_one_file_to_detect_them_all.py @@ -6,11 +6,14 @@ from keras.models import Model import struct import cv2 -from ..utils.weightreader import WeightReader -from ..utils.bbox import BoundBox -from ..utils.tools import preprocess_input, decode_netout -from ..utils.tools import correct_yolo_boxes, do_nms, draw_boxes -from ..model.yolo3 import make_yolov3_model +import sys +ROOT_DIR = os.path.abspath("../") +sys.path.append(ROOT_DIR) +from utils.weightreader import WeightReader +from utils.bbox import BoundBox +from utils.tools import preprocess_input, decode_netout +from utils.tools import correct_yolo_boxes, do_nms, draw_boxes +from model.yolo3 import make_yolov3_model np.set_printoptions(threshold=np.nan) os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" diff --git a/train_openimg.py b/train_openimg.py new file mode 100644 index 000000000..b772002ac --- /dev/null +++ b/train_openimg.py @@ -0,0 +1,282 @@ +#! /usr/bin/env python + +import argparse +import os +import numpy as np +import json +from .dataset.openimg import parse_openimg_annotation +from .model.yolo import create_yolov3_model, dummy_loss +from generator import BatchGenerator +from .utils.utils import normalize, evaluate, makedirs +from keras.callbacks import EarlyStopping, ReduceLROnPlateau +from keras.optimizers import Adam +from callbacks import CustomModelCheckpoint, CustomTensorBoard +from .utils.multi_gpu_model import multi_gpu_model +import tensorflow as tf +import keras +from keras.models import load_model + +def create_training_instances( + train_annot_file, + train_image_folder, + label_map, + train_cache, + valid_annot_folder, + valid_image_folder, + valid_cache, + labels, +): + # parse annotations of the training set + train_ints, train_labels = parse_openimg_annotation(train_annot_file, train_image_folder, label_map, train_cache, labels) + + # parse annotations of the validation set, if any, otherwise split the training set + if os.path.exists(valid_annot_folder): + valid_ints, valid_labels = parse_openimg_annotation(valid_annot_file, valid_image_folder, label_map, valid_cache, labels) + else: + print("valid_annot_folder not exists. Spliting the trainining set.") + + train_valid_split = int(0.8*len(train_ints)) + np.random.seed(0) + np.random.shuffle(train_ints) + np.random.seed() + + valid_ints = train_ints[train_valid_split:] + train_ints = train_ints[:train_valid_split] + + # compare the seen labels with the given labels in config.json + if len(labels) > 0: + overlap_labels = set(labels).intersection(set(train_labels.keys())) + + print('Seen labels: \t' + str(train_labels) + '\n') + print('Given labels: \t' + str(labels)) + + # return None, None, None if some given label is not in the dataset + if len(overlap_labels) < len(labels): + print('Some labels have no annotations! Please revise the list of labels in the config.json.') + return None, None, None + else: + print('No labels are provided. Train on all seen labels.') + print(train_labels) + labels = train_labels.keys() + + max_box_per_image = max([len(inst['object']) for inst in (train_ints + valid_ints)]) + + return train_ints, valid_ints, sorted(labels), max_box_per_image + +def create_callbacks(saved_weights_name, tensorboard_logs, model_to_save): + makedirs(tensorboard_logs) + + early_stop = EarlyStopping( + monitor = 'loss', + min_delta = 0.01, + patience = 5, + mode = 'min', + verbose = 1 + ) + checkpoint = CustomModelCheckpoint( + model_to_save = model_to_save, + filepath = saved_weights_name,# + '{epoch:02d}.h5', + monitor = 'loss', + verbose = 1, + save_best_only = True, + mode = 'min', + period = 1 + ) + reduce_on_plateau = ReduceLROnPlateau( + monitor = 'loss', + factor = 0.1, + patience = 2, + verbose = 1, + mode = 'min', + epsilon = 0.01, + cooldown = 0, + min_lr = 0 + ) + tensorboard = CustomTensorBoard( + log_dir = tensorboard_logs, + write_graph = True, + write_images = True, + ) + return [early_stop, checkpoint, reduce_on_plateau, tensorboard] + +def create_model( + nb_class, + anchors, + max_box_per_image, + max_grid, batch_size, + warmup_batches, + ignore_thresh, + multi_gpu, + saved_weights_name, + lr, + grid_scales, + obj_scale, + noobj_scale, + xywh_scale, + class_scale +): + if multi_gpu > 1: + with tf.device('/cpu:0'): + template_model, infer_model = create_yolov3_model( + nb_class = nb_class, + anchors = anchors, + max_box_per_image = max_box_per_image, + max_grid = max_grid, + batch_size = batch_size//multi_gpu, + warmup_batches = warmup_batches, + ignore_thresh = ignore_thresh, + grid_scales = grid_scales, + obj_scale = obj_scale, + noobj_scale = noobj_scale, + xywh_scale = xywh_scale, + class_scale = class_scale + ) + else: + template_model, infer_model = create_yolov3_model( + nb_class = nb_class, + anchors = anchors, + max_box_per_image = max_box_per_image, + max_grid = max_grid, + batch_size = batch_size, + warmup_batches = warmup_batches, + ignore_thresh = ignore_thresh, + grid_scales = grid_scales, + obj_scale = obj_scale, + noobj_scale = noobj_scale, + xywh_scale = xywh_scale, + class_scale = class_scale + ) + + # load the pretrained weight if exists, otherwise load the backend weight only + if os.path.exists(saved_weights_name): + print("\nLoading pretrained weights.\n") + template_model.load_weights(saved_weights_name) + else: + template_model.load_weights("backend.h5", by_name=True) + + if multi_gpu > 1: + train_model = multi_gpu_model(template_model, gpus=multi_gpu) + else: + train_model = template_model + + optimizer = Adam(lr=lr, clipnorm=0.001) + train_model.compile(loss=dummy_loss, optimizer=optimizer) + + return train_model, infer_model + +def _main_(args): + config_path = args.conf + + with open(config_path) as config_buffer: + config = json.loads(config_buffer.read()) + + ############################### + # Parse the annotations + ############################### + train_ints, valid_ints, labels, max_box_per_image = create_training_instances( + config['train']['train_annot_file'], + config['train']['train_image_folder'], + config['train']['label_map'], + config['train']['cache_name'], + config['valid']['valid_annot_file'], + config['valid']['valid_image_folder'], + config['valid']['cache_name'], + config['model']['labels'] + ) + print('\nTraining on: \t' + str(labels) + '\n') + + ############################### + # Create the generators + ############################### + train_generator = BatchGenerator( + instances = train_ints, + anchors = config['model']['anchors'], + labels = labels, + downsample = 32, # ratio between network input's size and network output's size, 32 for YOLOv3 + max_box_per_image = max_box_per_image, + batch_size = config['train']['batch_size'], + min_net_size = config['model']['min_input_size'], + max_net_size = config['model']['max_input_size'], + shuffle = True, + jitter = 0.3, + norm = normalize + ) + + valid_generator = BatchGenerator( + instances = valid_ints, + anchors = config['model']['anchors'], + labels = labels, + downsample = 32, # ratio between network input's size and network output's size, 32 for YOLOv3 + max_box_per_image = max_box_per_image, + batch_size = config['train']['batch_size'], + min_net_size = config['model']['min_input_size'], + max_net_size = config['model']['max_input_size'], + shuffle = True, + jitter = 0.0, + norm = normalize + ) + + ############################### + # Create the model + ############################### + if os.path.exists(config['train']['saved_weights_name']): + config['train']['warmup_epochs'] = 0 + warmup_batches = config['train']['warmup_epochs'] * (config['train']['train_times']*len(train_generator)) + + os.environ['CUDA_VISIBLE_DEVICES'] = config['train']['gpus'] + multi_gpu = len(config['train']['gpus'].split(',')) + + train_model, infer_model = create_model( + nb_class = len(labels), + anchors = config['model']['anchors'], + max_box_per_image = max_box_per_image, + max_grid = [config['model']['max_input_size'], config['model']['max_input_size']], + batch_size = config['train']['batch_size'], + warmup_batches = warmup_batches, + ignore_thresh = config['train']['ignore_thresh'], + multi_gpu = multi_gpu, + saved_weights_name = config['train']['saved_weights_name'], + lr = config['train']['learning_rate'], + grid_scales = config['train']['grid_scales'], + obj_scale = config['train']['obj_scale'], + noobj_scale = config['train']['noobj_scale'], + xywh_scale = config['train']['xywh_scale'], + class_scale = config['train']['class_scale'], + ) + + ############################### + # Kick off the training + ############################### + callbacks = create_callbacks(config['train']['saved_weights_name'], config['train']['tensorboard_dir'], infer_model) + + train_model.fit_generator( + generator = train_generator, + steps_per_epoch = len(train_generator) * config['train']['train_times'], + epochs = config['train']['nb_epochs'] + config['train']['warmup_epochs'], + verbose = 2 if config['train']['debug'] else 1, + callbacks = callbacks, + workers = 4, + max_queue_size = 8 + ) + + # make a GPU version of infer_model for evaluation + if multi_gpu > 1: + infer_model = load_model(config['train']['saved_weights_name']) + + ############################### + # Run the evaluation + ############################### + # compute mAP for all the classes + average_precisions = evaluate(infer_model, valid_generator) + + # print the score + for label, average_precision in average_precisions.items(): + print(labels[label] + ': {:.4f}'.format(average_precision)) + print('mAP: {:.4f}'.format(sum(average_precisions.values()) / len(average_precisions))) + +if __name__ == '__main__': + argparser = argparse.ArgumentParser(description='train and evaluate YOLO_v3 model on any dataset') + argparser.add_argument('-c', '--conf', help='path to configuration file') + + args = argparser.parse_args() + _main_(args)