-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.html
1964 lines (918 loc) · 94.5 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<html class="theme-next gemini use-motion" lang="en">
<head>
<meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">
<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />
<link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />
<link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />
<link href="/css/main.css?v=5.1.2" rel="stylesheet" type="text/css" />
<meta name="keywords" content="Hexo, NexT" />
<link rel="shortcut icon" type="image/x-icon" href="/favicon.png?v=5.1.2" />
<meta name="description" content="God helps those who help themselves.">
<meta property="og:type" content="website">
<meta property="og:title" content="Howard的學習之路">
<meta property="og:url" content="http://yoursite.com/index.html">
<meta property="og:site_name" content="Howard的學習之路">
<meta property="og:description" content="God helps those who help themselves.">
<meta property="og:locale" content="en">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Howard的學習之路">
<meta name="twitter:description" content="God helps those who help themselves.">
<script type="text/javascript" id="hexo.configurations">
var NexT = window.NexT || {};
var CONFIG = {
root: '/',
scheme: 'Gemini',
sidebar: {"position":"left","display":"post","offset":12,"offset_float":12,"b2t":false,"scrollpercent":false,"onmobile":false},
fancybox: true,
tabs: true,
motion: true,
duoshuo: {
userId: '0',
author: 'Author'
},
algolia: {
applicationID: '',
apiKey: '',
indexName: '',
hits: {"per_page":10},
labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
}
};
</script>
<link rel="canonical" href="http://yoursite.com/"/>
<title>Howard的學習之路</title>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-111619338-1', 'auto');
ga('send', 'pageview');
</script>
</head>
<body itemscope itemtype="http://schema.org/WebPage" lang="en">
<div class="container sidebar-position-left
page-home
">
<div class="headband"></div>
<header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
<div class="header-inner"><div class="site-brand-wrapper">
<div class="site-meta ">
<div class="custom-logo-site-title">
<a href="/" class="brand" rel="start">
<span class="logo-line-before"><i></i></span>
<span class="site-title">Howard的學習之路</span>
<span class="logo-line-after"><i></i></span>
</a>
</div>
<p class="site-subtitle"></p>
</div>
<div class="site-nav-toggle">
<button>
<span class="btn-bar"></span>
<span class="btn-bar"></span>
<span class="btn-bar"></span>
</button>
</div>
</div>
<nav class="site-nav">
<ul id="menu" class="menu">
<li class="menu-item menu-item-home">
<a href="/" rel="section">
<i class="menu-item-icon fa fa-fw fa-home"></i> <br />
Home
</a>
</li>
<li class="menu-item menu-item-archives">
<a href="/archives/" rel="section">
<i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
Archives
</a>
</li>
<li class="menu-item menu-item-tags">
<a href="/tags/" rel="section">
<i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
Tags
</a>
</li>
<li class="menu-item menu-item-about">
<a href="/about/" rel="section">
<i class="menu-item-icon fa fa-fw fa-user"></i> <br />
About
</a>
</li>
</ul>
</nav>
</div>
</header>
<main id="main" class="main">
<div class="main-inner">
<div class="content-wrap">
<div id="content" class="content">
<section id="posts" class="posts-expand">
<article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
<div class="post-block">
<link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/12/28/Hualos-在Keras中動態監視training過程/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Howard Chang">
<meta itemprop="description" content="">
<meta itemprop="image" content="/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Howard的學習之路">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2017/12/28/Hualos-在Keras中動態監視training過程/" itemprop="url">Hualos - 在Keras中動態監視training過程</a></h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Post created" itemprop="dateCreated datePublished" datetime="2017-12-28T12:48:43+08:00">
2017-12-28
</time>
</span>
<span class="post-comments-count">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-comment-o"></i>
</span>
<a href="/2017/12/28/Hualos-在Keras中動態監視training過程/#comments" itemprop="discussionUrl">
<span class="post-comments-count disqus-comment-count"
data-disqus-identifier="2017/12/28/Hualos-在Keras中動態監視training過程/" itemprop="commentCount"></span>
</a>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="Hualos-在Keras中動態監視training過程"><a href="#Hualos-在Keras中動態監視training過程" class="headerlink" title="Hualos - 在Keras中動態監視training過程"></a>Hualos - 在Keras中動態監視training過程</h1><p>Keras本身就有提供<code>callbacks</code>機制，可以讓我們在training過程中，看到一些資訊。<br>這邊要教如何使用<code>Hualos</code>，讓我們在Web上看到training時的acc、loss等變化。</p>
<h2 id="使用RemoteMonitor"><a href="#使用RemoteMonitor" class="headerlink" title="使用RemoteMonitor"></a>使用RemoteMonitor</h2><p>我們可以直接用<code>callbacks.RemoteMonitor()</code>將training時的acc、loss、val_acc、val_loss，POST到我們的Server上。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> keras <span class="keyword">import</span> callbacks</span><br><span class="line">remote = callbacks.RemoteMonitor(root=<span class="string">'http://localhost:9000'</span>)</span><br><span class="line"></span><br><span class="line">model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, validation_data=(X_test, Y_test), callbacks=[remote])</span><br></pre></td></tr></table></figure>
<p>當然網頁要自己寫太麻煩了，這邊使用<code>Hualos</code>來替我們完成。</p>
<h2 id="安裝Hualos所需套件"><a href="#安裝Hualos所需套件" class="headerlink" title="安裝Hualos所需套件"></a>安裝Hualos所需套件</h2><p>由於<code>Hualos</code>需要用到以下兩個Python套件，可以透過<code>pip</code>直接安裝</p>
<ul>
<li>Flask</li>
<li>gevent</li>
</ul>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ pip install flask gevent</span><br></pre></td></tr></table></figure>
<h2 id="下載Hualos"><a href="#下載Hualos" class="headerlink" title="下載Hualos"></a>下載Hualos</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ git clone https://github.com/fchollet/hualos.git</span><br></pre></td></tr></table></figure>
<h2 id="執行Hualos"><a href="#執行Hualos" class="headerlink" title="執行Hualos"></a>執行Hualos</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ python hualos/api.py</span><br></pre></td></tr></table></figure>
<p>開啟瀏覽器，進入<code>localhost:9000</code></p>
<h2 id="Training"><a href="#Training" class="headerlink" title="Training"></a>Training</h2><p>使用<code>mnist</code>為範例，完整程式碼如下</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> keras</span><br><span class="line"><span class="keyword">from</span> keras.datasets <span class="keyword">import</span> mnist</span><br><span class="line"><span class="keyword">from</span> keras.models <span class="keyword">import</span> Sequential</span><br><span class="line"><span class="keyword">from</span> keras.layers <span class="keyword">import</span> Dense, Dropout</span><br><span class="line"><span class="keyword">from</span> keras.optimizers <span class="keyword">import</span> RMSprop</span><br><span class="line"><span class="keyword">from</span> keras <span class="keyword">import</span> callbacks</span><br><span class="line"></span><br><span class="line">batch_size = <span class="number">128</span></span><br><span class="line">num_classes = <span class="number">10</span></span><br><span class="line">epochs = <span class="number">20</span></span><br><span class="line"></span><br><span class="line">(x_train, y_train), (x_test, y_test) = mnist.load_data()</span><br><span class="line"></span><br><span class="line">x_train = x_train.reshape(<span class="number">60000</span>, <span class="number">784</span>)</span><br><span class="line">x_test = x_test.reshape(<span class="number">10000</span>, <span class="number">784</span>)</span><br><span class="line">x_train = x_train.astype(<span class="string">'float32'</span>)</span><br><span class="line">x_test = x_test.astype(<span class="string">'float32'</span>)</span><br><span class="line">x_train /= <span class="number">255</span></span><br><span class="line">x_test /= <span class="number">255</span></span><br><span class="line">print(x_train.shape[<span class="number">0</span>], <span class="string">'train samples'</span>)</span><br><span class="line">print(x_test.shape[<span class="number">0</span>], <span class="string">'test samples'</span>)</span><br><span class="line"></span><br><span class="line">y_train = keras.utils.to_categorical(y_train, num_classes)</span><br><span class="line">y_test = keras.utils.to_categorical(y_test, num_classes)</span><br><span class="line"></span><br><span class="line">model = Sequential()</span><br><span class="line">model.add(Dense(<span class="number">512</span>, activation=<span class="string">'relu'</span>, input_shape=(<span class="number">784</span>,)))</span><br><span class="line">model.add(Dropout(<span class="number">0.2</span>))</span><br><span class="line">model.add(Dense(<span class="number">512</span>, activation=<span class="string">'relu'</span>))</span><br><span class="line">model.add(Dropout(<span class="number">0.2</span>))</span><br><span class="line">model.add(Dense(num_classes, activation=<span class="string">'softmax'</span>))</span><br><span class="line"></span><br><span class="line">model.summary()</span><br><span class="line"></span><br><span class="line">model.compile(loss=<span class="string">'categorical_crossentropy'</span>,</span><br><span class="line"> optimizer=RMSprop(),</span><br><span class="line"> metrics=[<span class="string">'accuracy'</span>])</span><br><span class="line"></span><br><span class="line">remote = callbacks.RemoteMonitor(root=<span class="string">'http://localhost:9000'</span>)</span><br><span class="line"></span><br><span class="line">model.fit(x_train, y_train,</span><br><span class="line"> batch_size=batch_size,</span><br><span class="line"> epochs=epochs,</span><br><span class="line"> verbose=<span class="number">1</span>,</span><br><span class="line"> validation_data=(x_test, y_test),</span><br><span class="line"> callbacks=[remote])</span><br><span class="line"></span><br><span class="line">score = model.evaluate(x_test, y_test, verbose=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">print(<span class="string">'Test loss:'</span>, score[<span class="number">0</span>])</span><br><span class="line">print(<span class="string">'Test accuracy:'</span>, score[<span class="number">1</span>])</span><br></pre></td></tr></table></figure>
<p>回到網頁就能即時看到training時的acc、loss、val_acc、val_loss。</p>
<p><img src="/2017/12/28/Hualos-在Keras中動態監視training過程/images/result.png" alt=""></p>
<h2 id="參考"><a href="#參考" class="headerlink" title="參考"></a>參考</h2><p><a href="https://github.com/fchollet/hualos" target="_blank" rel="noopener">https://github.com/fchollet/hualos</a></p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</div>
</article>
<article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
<div class="post-block">
<link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/12/26/Hyperas-在Keras中自動選擇超參數/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Howard Chang">
<meta itemprop="description" content="">
<meta itemprop="image" content="/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Howard的學習之路">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2017/12/26/Hyperas-在Keras中自動選擇超參數/" itemprop="url">Hyperas - 在Keras中自動選擇超參數</a></h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Post created" itemprop="dateCreated datePublished" datetime="2017-12-26T15:50:25+08:00">
2017-12-26
</time>
</span>
<span class="post-comments-count">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-comment-o"></i>
</span>
<a href="/2017/12/26/Hyperas-在Keras中自動選擇超參數/#comments" itemprop="discussionUrl">
<span class="post-comments-count disqus-comment-count"
data-disqus-identifier="2017/12/26/Hyperas-在Keras中自動選擇超參數/" itemprop="commentCount"></span>
</a>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="Hyperas-在Keras中自動選擇超參數"><a href="#Hyperas-在Keras中自動選擇超參數" class="headerlink" title="Hyperas - 在Keras中自動選擇超參數"></a>Hyperas - 在Keras中自動選擇超參數</h1><p>deep learning做到後面都剩下調參數<br>而參數又不是那麼容易調整，是個廢力又廢時的工作<br>這邊將介紹透過Hyperas這個套件，自動選擇符合model最好的參數</p>
<h2 id="安裝Hyperas"><a href="#安裝Hyperas" class="headerlink" title="安裝Hyperas"></a>安裝Hyperas</h2><p>使用<code>pip</code>進行安裝</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ pip install hyperas</span><br></pre></td></tr></table></figure>
<h2 id="Import-Hyperas"><a href="#Import-Hyperas" class="headerlink" title="Import Hyperas"></a>Import Hyperas</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> hyperopt <span class="keyword">import</span> Trials, STATUS_OK, tpe</span><br><span class="line"><span class="keyword">from</span> hyperas <span class="keyword">import</span> optim</span><br><span class="line"><span class="keyword">from</span> hyperas.distributions <span class="keyword">import</span> choice, uniform</span><br></pre></td></tr></table></figure>
<h2 id="Import-Keras"><a href="#Import-Keras" class="headerlink" title="Import Keras"></a>Import Keras</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> keras.models <span class="keyword">import</span> Sequential</span><br><span class="line"><span class="keyword">from</span> keras.layers.core <span class="keyword">import</span> Dense, Dropout, Activation</span><br><span class="line"><span class="keyword">from</span> keras.optimizers <span class="keyword">import</span> RMSprop</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> keras.datasets <span class="keyword">import</span> mnist</span><br><span class="line"><span class="keyword">from</span> keras.utils <span class="keyword">import</span> np_utils</span><br></pre></td></tr></table></figure>
<p>之後我們會依序</p>
<ol>
<li>定義Data</li>
<li>定義Model</li>
<li>Optimize model hyperparameters</li>
</ol>
<h2 id="定義Data"><a href="#定義Data" class="headerlink" title="定義Data"></a>定義Data</h2><p>使用<code>MNIST</code>的data</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">data</span><span class="params">()</span>:</span></span><br><span class="line"> (X_train, y_train), (X_test, y_test) = mnist.load_data()</span><br><span class="line"> X_train = X_train.reshape(<span class="number">60000</span>, <span class="number">784</span>)</span><br><span class="line"> X_test = X_test.reshape(<span class="number">10000</span>, <span class="number">784</span>)</span><br><span class="line"> X_train = X_train.astype(<span class="string">'float32'</span>)</span><br><span class="line"> X_test = X_test.astype(<span class="string">'float32'</span>)</span><br><span class="line"> X_train /= <span class="number">255</span></span><br><span class="line"> X_test /= <span class="number">255</span></span><br><span class="line"> nb_classes = <span class="number">10</span></span><br><span class="line"> Y_train = np_utils.to_categorical(y_train, nb_classes)</span><br><span class="line"> Y_test = np_utils.to_categorical(y_test, nb_classes)</span><br><span class="line"> <span class="keyword">return</span> X_train, Y_train, X_test, Y_test</span><br></pre></td></tr></table></figure>
<h2 id="定義Model"><a href="#定義Model" class="headerlink" title="定義Model"></a>定義Model</h2><p>這邊除了定義Model外，還需完成training及testing，所以需把data傳進來<br>最後回傳一個dictionary，其中包含：</p>
<ul>
<li>loss: Hyperas會去選擇最小值的model</li>
<li>status: 直接回傳<code>STATUS_OK</code></li>
<li>model: 可不回傳(option)</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">create_model</span><span class="params">(X_train, Y_train, X_test, Y_test)</span>:</span></span><br><span class="line"> model = Sequential()</span><br><span class="line"> model.add(Dense(<span class="number">512</span>, input_shape=(<span class="number">784</span>,)))</span><br><span class="line"> model.add(Activation(<span class="string">'relu'</span>))</span><br><span class="line"> model.add(Dropout({{uniform(<span class="number">0</span>, <span class="number">1</span>)}}))</span><br><span class="line"> model.add(Dense({{choice([<span class="number">256</span>, <span class="number">512</span>, <span class="number">1024</span>])}}))</span><br><span class="line"> model.add(Activation(<span class="string">'relu'</span>))</span><br><span class="line"> model.add(Dropout({{uniform(<span class="number">0</span>, <span class="number">1</span>)}}))</span><br><span class="line"> model.add(Dense(<span class="number">10</span>))</span><br><span class="line"> model.add(Activation(<span class="string">'softmax'</span>))</span><br><span class="line"></span><br><span class="line"> rms = RMSprop()</span><br><span class="line"> model.compile(loss=<span class="string">'categorical_crossentropy'</span>, optimizer=rms, metrics=[<span class="string">'accuracy'</span>])</span><br><span class="line"> </span><br><span class="line"> model.fit(X_train, Y_train,</span><br><span class="line"> batch_size={{choice([<span class="number">64</span>, <span class="number">128</span>])}},</span><br><span class="line"> nb_epoch=<span class="number">1</span>,</span><br><span class="line"> verbose=<span class="number">2</span>,</span><br><span class="line"> validation_data=(X_test, Y_test))</span><br><span class="line"> score, acc = model.evaluate(X_test, Y_test, verbose=<span class="number">0</span>)</span><br><span class="line"> print(<span class="string">'Test accuracy:'</span>, acc)</span><br><span class="line"> <span class="keyword">return</span> {<span class="string">'loss'</span>: -acc, <span class="string">'status'</span>: STATUS_OK, <span class="string">'model'</span>: model}</span><br></pre></td></tr></table></figure>
<p>原本Dropout需要傳入一個0-1的機率<br>但我們這邊不直接指定一個數字<br>而是透過<code>uniform</code>幫我們產生一個0-1的數字</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">model.add(Dropout({{uniform(<span class="number">0</span>, <span class="number">1</span>)}}))</span><br></pre></td></tr></table></figure>
<p>Dense擇是透過<code>choice</code><br>傳入我們要哪些值</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">model.add(Dense({{choice([<span class="number">256</span>, <span class="number">512</span>, <span class="number">1024</span>])}}))</span><br></pre></td></tr></table></figure>
<p>最後回傳的dictionary<br>我們目標是選擇最高的accuracy的model<br>但因為Huperas他會去選擇<code>loss</code>這個value <strong>最小的</strong> 的model<br>所以將accuracy直接變 <strong>負號</strong><br>再丟給<code>loss</code></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">return</span> {<span class="string">'loss'</span>: -acc, <span class="string">'status'</span>: STATUS_OK, <span class="string">'model'</span>: model}</span><br></pre></td></tr></table></figure>
<h2 id="Optimize-model-hyperparameters"><a href="#Optimize-model-hyperparameters" class="headerlink" title="Optimize model hyperparameters"></a>Optimize model hyperparameters</h2><p>最後透過<code>optim.minimize()</code>來找出最好的model</p>
<ul>
<li>model: 我們定義的model</li>
<li>data: 我們定義的data</li>
<li>algo: 使用TPE algorithm</li>
<li>max_evals: evaluation次數</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">X_train, Y_train, X_test, Y_test = data()</span><br><span class="line"></span><br><span class="line">best_run, best_model = optim.minimize(model=create_model,</span><br><span class="line"> data=data,</span><br><span class="line"> algo=tpe.suggest,</span><br><span class="line"> max_evals=<span class="number">5</span>,</span><br><span class="line"> trials=Trials())</span><br><span class="line"></span><br><span class="line">print(<span class="string">"Evalutation of best performing model:"</span>)</span><br><span class="line">print(best_model.evaluate(X_test, Y_test))</span><br><span class="line">print(best_run)</span><br></pre></td></tr></table></figure>
<p><code>optim.minimize()</code>會回傳</p>
<ul>
<li>best_run: 最好的參數組合</li>
<li>best_model: 最好的model</li>
</ul>
<h2 id="最後"><a href="#最後" class="headerlink" title="最後"></a>最後</h2><ol>
<li>Hyperas好像跟註解很不合，在跑程式時需把註解都刪掉，以免發生錯誤</li>
<li>如果是使用jupyter notebook需在<code>optim.minimize()</code>多加入<code>notebook_name</code>這個參數且設定為<code>ipynb</code>的檔名，假如目前為<code>Untitled.ipynb</code>就設定為：</li>
</ol>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">best_run, best_model = optim.minimize(model=create_model,</span><br><span class="line"> data=data,</span><br><span class="line"> algo=tpe.suggest,</span><br><span class="line"> max_evals=<span class="number">5</span>,</span><br><span class="line"> trials=Trials(),</span><br><span class="line"> notebook_name=<span class="string">'Untitled'</span>)</span><br></pre></td></tr></table></figure>
<h2 id="參考"><a href="#參考" class="headerlink" title="參考"></a>參考</h2><p><a href="https://github.com/maxpumperla/hyperas" target="_blank" rel="noopener">https://github.com/maxpumperla/hyperas</a></p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</div>
</article>
<article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
<div class="post-block">
<link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/12/25/FB-Messenger-Bot-建置教學/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Howard Chang">
<meta itemprop="description" content="">
<meta itemprop="image" content="/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Howard的學習之路">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2017/12/25/FB-Messenger-Bot-建置教學/" itemprop="url">FB Messenger Bot 建置教學</a></h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Post created" itemprop="dateCreated datePublished" datetime="2017-12-25T22:08:29+08:00">
2017-12-25
</time>
</span>
<span class="post-comments-count">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-comment-o"></i>
</span>
<a href="/2017/12/25/FB-Messenger-Bot-建置教學/#comments" itemprop="discussionUrl">
<span class="post-comments-count disqus-comment-count"
data-disqus-identifier="2017/12/25/FB-Messenger-Bot-建置教學/" itemprop="commentCount"></span>
</a>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="FB-Messenger-Bot-建置教學"><a href="#FB-Messenger-Bot-建置教學" class="headerlink" title="FB Messenger Bot 建置教學"></a>FB Messenger Bot 建置教學</h1><p>此教學為建置一個FB粉絲團自動回化的機器人之環境</p>
<h2 id="Facebook-Developer的APP建立"><a href="#Facebook-Developer的APP建立" class="headerlink" title="Facebook Developer的APP建立"></a>Facebook Developer的APP建立</h2><p>既然是要寫這項功能自然就是要開啟FB開發者的功能<br>到<a href="https://developers.facebook.com/" target="_blank" rel="noopener">https://developers.facebook.com/</a></p>
<p>點擊右上角的 <strong>「我的應用程式」</strong> -> <strong>「新增應用程式」</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/1.png" alt=""></p>
<p>輸入名稱及選擇類別後就可以建立應用程式了<br>並在Messenger選項點選 <strong>「開始使用」</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/2.png" alt=""></p>
<h2 id="建立粉絲專頁存取權杖"><a href="#建立粉絲專頁存取權杖" class="headerlink" title="建立粉絲專頁存取權杖"></a>建立粉絲專頁存取權杖</h2><p>選擇粉絲團及產生一組權杖 <strong>(pageAccessToken)</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/3.png" alt=""></p>
<h2 id="查看應用程式密鑰"><a href="#查看應用程式密鑰" class="headerlink" title="查看應用程式密鑰"></a>查看應用程式密鑰</h2><p>點選左上角的主控版查看應用程式密鑰 <strong>(appSecret)</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/4.png" alt=""></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/5.png" alt=""></p>
<h2 id="建立HTTP-Server"><a href="#建立HTTP-Server" class="headerlink" title="建立HTTP Server"></a>建立HTTP Server</h2><p>這邊使用免費的虛擬主機 <a href="https://c9.io/" target="_blank" rel="noopener">https://c9.io/</a><br>辦完帳號就能建立一個新的workspaces<br>輸入name，且一定要選 <strong>Public</strong> 的權限<br>Git clone的URL輸入 <a href="https://github.com/fbsamples/messenger-platform-samples.git" target="_blank" rel="noopener">https://github.com/fbsamples/messenger-platform-samples.git</a><br>最後template選擇 <strong>Node.js</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/6.png" alt=""></p>
<p>建立完成後在下方Terminal依序輸入下列指令</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">$ cd node/</span><br><span class="line">$ npm install</span><br></pre></td></tr></table></figure>
<p>成功時會看到產生了 <strong>node_modules</strong> 資料夾<br>這時開啟 <strong>node/config/default.json</strong> 這個檔案</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">{</span><br><span class="line"> "appSecret": "",</span><br><span class="line"> "pageAccessToken": "",</span><br><span class="line"> "validationToken": "",</span><br><span class="line"> "serverURL": ""</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<ul>
<li>appSecret: 這邊填入應用程式密鑰</li>
<li>pageAccessToken: 粉絲專頁存取權杖</li>
<li>validationToken: 自己可以任意輸入，但要需要記住，等等下面會用到</li>
<li>serverURL: 輸入https:// <strong>專案名稱</strong> –<strong>帳號名稱</strong> .c9users.io</li>
</ul>
<p>完成後存檔並在下面Terminal輸入下列指令來啟動Server</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">$ node app.js</span><br></pre></td></tr></table></figure>
<p>成功時會看到 <strong>「Node app is running on port 8080」</strong></p>
<h2 id="設定Webhooks"><a href="#設定Webhooks" class="headerlink" title="設定Webhooks"></a>設定Webhooks</h2><p>回到FB Developer的頁面，點擊右邊的 <strong>「設定Webhooks」</strong></p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/7.png" alt=""></p>
<p>在回呼網址輸入https:// <strong>專案名稱</strong> –<strong>帳號名稱</strong> .c9users.io/webhook<br>驗證權杖這邊輸入上面json裡 <strong>validationToken</strong> 的值<br>下面的訂閱欄位依自己需求勾選</p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/8.png" alt=""></p>
<p>儲存完後就可以選訂閱的粉絲團</p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/9.png" alt=""></p>
<h2 id="測試Bot"><a href="#測試Bot" class="headerlink" title="測試Bot"></a>測試Bot</h2><p>到粉絲團頁面，選擇「更多」 -> 「以粉絲專頁訪客的身分檢視」<br>就可以傳送訊息給粉絲團了</p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/10.png" alt=""></p>
<p>看到Bot成功回你輸入的訊息就代表成功囉</p>
<p><img src="/2017/12/25/FB-Messenger-Bot-建置教學/11.png" alt=""></p>
<h2 id="客制化回話"><a href="#客制化回話" class="headerlink" title="客制化回話"></a>客制化回話</h2><p>主要修改app.js這檔案</p>
<ul>
<li>收到訊息的事件在 function receivedMessage(event)</li>
</ul>
<figure class="highlight javascript"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment"> * Message Event</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * This event is called when a message is sent to your page. The 'message' </span></span><br><span class="line"><span class="comment"> * object format can vary depending on the kind of message that was received.</span></span><br><span class="line"><span class="comment"> * Read more at https://developers.facebook.com/docs/messenger-platform/webhook-reference/message-received</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> * For this example, we're going to echo any text that we get. If we get some </span></span><br><span class="line"><span class="comment"> * special keywords ('button', 'generic', 'receipt'), then we'll send back</span></span><br><span class="line"><span class="comment"> * examples of those bubbles to illustrate the special message bubbles we've </span></span><br><span class="line"><span class="comment"> * created. If we receive a message with an attachment (image, video, audio), </span></span><br><span class="line"><span class="comment"> * then we'll simply confirm that we've received the attachment.</span></span><br><span class="line"><span class="comment"> * </span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">receivedMessage</span>(<span class="params">event</span>) </span>{</span><br><span class="line"> ...</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<ul>
<li>傳送訊息的方法在 function sendTextMessage(recipientId, messageText)</li>
</ul>
<figure class="highlight javascript"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">/*</span></span><br><span class="line"><span class="comment"> * Send a text message using the Send API.</span></span><br><span class="line"><span class="comment"> *</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">sendTextMessage</span>(<span class="params">recipientId, messageText</span>) </span>{</span><br><span class="line"> ...</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p>基本上只要用這兩個Function就能客制化自己的機器人囉~</p>
<h2 id="參考"><a href="#參考" class="headerlink" title="參考"></a>參考</h2><p><a href="http://animabeautifullife.blogspot.tw/2016/06/facebook-messenger-api.html" target="_blank" rel="noopener">http://animabeautifullife.blogspot.tw/2016/06/facebook-messenger-api.html</a></p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</div>
</article>
<article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
<div class="post-block">
<link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/12/25/緩衝區溢位原理/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Howard Chang">
<meta itemprop="description" content="">
<meta itemprop="image" content="/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Howard的學習之路">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2017/12/25/緩衝區溢位原理/" itemprop="url">緩衝區溢位原理</a></h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Post created" itemprop="dateCreated datePublished" datetime="2017-12-25T21:43:38+08:00">
2017-12-25
</time>
</span>
<span class="post-comments-count">
<span class="post-meta-divider">|</span>
<span class="post-meta-item-icon">
<i class="fa fa-comment-o"></i>
</span>
<a href="/2017/12/25/緩衝區溢位原理/#comments" itemprop="discussionUrl">
<span class="post-comments-count disqus-comment-count"
data-disqus-identifier="2017/12/25/緩衝區溢位原理/" itemprop="commentCount"></span>
</a>
</span>
</div>
</header>
<div class="post-body" itemprop="articleBody">
<h1 id="緩衝區溢位原理"><a href="#緩衝區溢位原理" class="headerlink" title="緩衝區溢位原理"></a>緩衝區溢位原理</h1><h2 id="目的"><a href="#目的" class="headerlink" title="目的"></a>目的</h2><p>指的是利用程式的漏洞，向緩衝衝區寫入使溢位的值，可用來達到以下目的：</p>
<ul>
<li>程序破壞</li>
<li>控制程式流程</li>
<li>取得系統的控制權</li>
</ul>
<p>以下將透過編寫簡單的程式，並輸入過長字串讓程序當掉，以了解緩衝區溢位的原理。</p>
<h2 id="環境介紹"><a href="#環境介紹" class="headerlink" title="環境介紹"></a>環境介紹</h2><ul>
<li>OS：Windows XP_x86 (SP3)</li>
<li>IDE：VC++ 6.0</li>
</ul>
<h2 id="事前準備及利用緩衝區漏洞"><a href="#事前準備及利用緩衝區漏洞" class="headerlink" title="事前準備及利用緩衝區漏洞"></a>事前準備及利用緩衝區漏洞</h2><p>這裡以C寫一段存在緩衝區漏洞的程式</p>
<figure class="highlight c"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string"><stdio.h></span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string"><string.h></span></span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">char</span> name[] = <span class="string">"HAO-WEI"</span>;</span><br><span class="line"> </span><br><span class="line"><span class="function"><span class="keyword">int</span> <span class="title">main</span><span class="params">()</span> </span>{</span><br><span class="line"> <span class="keyword">char</span> buffer[<span class="number">8</span>];</span><br><span class="line"> <span class="built_in">strcpy</span>(buffer, name);</span><br><span class="line"> <span class="built_in">printf</span>(<span class="string">"%s \n"</span>, buffer);</span><br><span class="line"> </span><br><span class="line"> <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p>這段程式是先宣告name變數資料型態為Char的陣列，並設值為”HAO-WEI”，<br>並在主程式中，宣告buffer變數一樣為Char的陣列的資料型態，設定與name變數相同長度，因為HAO-WEI後面還有一個’\0′<br>使用strcpy函數將name的字串複製給buffer，最後再輸出。</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1.jpg" alt=""></p>
<p>以上是有存在緩衝區溢位的程式碼，這邊思考一下，如果name的長度比8更長，或是buffer長度比8小，讓他真的發生緩衝區溢位，那麼會發生什麼事呢？<br>這邊將name內容設為HAO-WEIHAO-WEI，長度為15，並編譯執行</p>
<figure class="highlight c"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string"><stdio.h></span></span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">include</span> <span class="meta-string"><string.h></span></span></span><br><span class="line"> </span><br><span class="line"><span class="keyword">char</span> name[] = <span class="string">"HAO-WEI"</span>;</span><br><span class="line"> </span><br><span class="line"><span class="function"><span class="keyword">int</span> <span class="title">main</span><span class="params">()</span> </span>{</span><br><span class="line"> <span class="keyword">char</span> buffer[<span class="number">8</span>];</span><br><span class="line"> <span class="built_in">strcpy</span>(buffer, name);</span><br><span class="line"> <span class="built_in">printf</span>(<span class="string">"%s \n"</span>, buffer);</span><br><span class="line"> </span><br><span class="line"> <span class="keyword">return</span> <span class="number">0</span>;</span><br><span class="line">}</span><br></pre></td></tr></table></figure>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-b.jpg" alt=""></p>
<p>程序雖然成功的輸出HAO-WEIHAO-WEI，但隨即就當掉了。<br>緩衝區不就只是滿了，為什麼會發生這樣的事情呢？</p>
<h2 id="分析說明"><a href="#分析說明" class="headerlink" title="分析說明"></a>分析說明</h2><p>要解釋這個問題時，先講解程式在執行時Stack運作原理<br>程式在直行時會有一塊記憶體是存放Stack的部分<br>當進入副程式時，系統會給予這副程式一塊可用記憶體，稱為Stack Frame(如下圖)</p>
<p><img src="/2017/12/25/緩衝區溢位原理/stack.jpg" alt=""></p>
<p>其中epb跟esp都是暫存器的值</p>
<ul>
<li>epb表示Stack底端</li>
<li>esp表示Stack頂端</li>
</ul>
<p>每一個Stack Frame都有屬於自己的esp跟ebp<br>所以只要把epb的值+4就可以拿到這個Function的變數2的值<br>epb的值+8也就可以拿到變數1的值<br>而最上面綠色那塊則是這Function可用的其他記憶體空間<br>另外也會把這個Function完成後<br>等等要執行下一個指令的位置寫到Return Address這裡<br>也會把呼叫這個Function的舊ebp保存起來<br>以供結束此Function能夠回復上一個Function的ebp<br>這樣Function一直不斷呼叫，記憶體不斷的往上疊，也不會亂掉，因為每個Function只需在意他自己的ebp跟esp</p>
<p>這邊用OllyDbg來進行動態分析，以了解程式的組合語言以及執行時的Stack變化</p>
<h3 id="進入FUNCTION前"><a href="#進入FUNCTION前" class="headerlink" title="進入FUNCTION前"></a>進入FUNCTION前</h3><p>我們先把程式停在進入Ｍain()這個Function前<br>另外也需特別注意下一個指令的位置00401699<br>結束完Main()時會繼續往這裡走，所以需要把這個位置寫到Stack中(等等下面圖片會看到)</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">00401694 CALL BufferOv.00401005 // 表示CALL Main()</span><br></pre></td></tr></table></figure>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-c.png" alt=""></p>
<p>以及他的Stack狀況</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-d.png" alt=""></p>
<p>可以知道目前esp為0012FF88</p>
<h3 id="進入FUNCTION後"><a href="#進入FUNCTION後" class="headerlink" title="進入FUNCTION後"></a>進入FUNCTION後</h3><p>觀察一下Stack狀態<br>就可以發現目前的esp變為0012FF24<br>而ebp為0012FF80 其值為0012FFC0(上一個Function的ebp)<br>而0012FF84(ebp + 4) 其值為00401699(為Main()執行完要回去的位置，也稱Return Address)<br>而這個Main()所用記憶體也就是0012FF24 ~ 0012FF7C這些空間</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-h.png" alt=""></p>
<h3 id="執行STRCPY前"><a href="#執行STRCPY前" class="headerlink" title="執行STRCPY前"></a>執行STRCPY前</h3><p>我們來看一下Main()裡面寫到的strcpy這Function的C語言以及其對應組合語言的部分</p>
<figure class="highlight c"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">strcpy</span>(buffer, name);</span><br></pre></td></tr></table></figure>
<p>strcpy的API</p>
<figure class="highlight c"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">char</span> * <span class="title">strcpy</span> <span class="params">( <span class="keyword">char</span> * destination, <span class="keyword">const</span> <span class="keyword">char</span> * source )</span></span>;</span><br></pre></td></tr></table></figure>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-g.png" alt=""></p>
<p>可以知道他把”HAO-WEI”這些字串拷貝到[ebp – 8]也就是0012FF78這個位置上</p>
<h3 id="執行STRCPY後"><a href="#執行STRCPY後" class="headerlink" title="執行STRCPY後"></a>執行STRCPY後</h3><p>我們來看看0012FF78這個位置的值發生什麼變化</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-i.png" alt=""></p>
<p>已經被填入”HAO-WEI”的字串了，看起來沒什麼問題<br>那如果是填入過長字串如”HAO-WEIHAO-WEI”會怎樣呢？我們來看看</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-j.png" alt=""></p>
<p>結果發現0012FF84原本放Main()執行完要去的位置被覆寫掉了，變成00004945<br>意謂著等等結束Main()會跳到00004945位置繼續執行指令，而這個位置如果是空的，程式自然就會當掉，發生以下情形</p>
<p><img src="/2017/12/25/緩衝區溢位原理/1-1-k.png" alt=""></p>
<p>很明顯的圖片上出現的0x00004945就是這個意思</p>
<h2 id="最後"><a href="#最後" class="headerlink" title="最後"></a>最後</h2><p>換個角度想，如果覆寫掉的這個位置是存在的，就樣是不是就能控制程式流程，隨心所欲的跳到想要的位置呢？</p>
</div>
<footer class="post-footer">
<div class="post-eof"></div>
</footer>
</div>
</article>
<article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
<div class="post-block">
<link itemprop="mainEntityOfPage" href="http://yoursite.com/2017/12/25/Spark與Flume之環境建置/">
<span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
<meta itemprop="name" content="Howard Chang">
<meta itemprop="description" content="">
<meta itemprop="image" content="/avatar.jpg">
</span>
<span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
<meta itemprop="name" content="Howard的學習之路">
</span>
<header class="post-header">
<h1 class="post-title" itemprop="name headline">
<a class="post-title-link" href="/2017/12/25/Spark與Flume之環境建置/" itemprop="url">Spark與Flume之環境建置</a></h1>
<div class="post-meta">
<span class="post-time">
<span class="post-meta-item-icon">
<i class="fa fa-calendar-o"></i>
</span>
<span class="post-meta-item-text">Posted on</span>
<time title="Post created" itemprop="dateCreated datePublished" datetime="2017-12-25T21:19:32+08:00">
2017-12-25
</time>