-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.html
430 lines (329 loc) · 42.9 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>Sicilienne</title><meta name="author" content="Zoris"><meta name="copyright" content="Zoris"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta property="og:type" content="website">
<meta property="og:title" content="Sicilienne">
<meta property="og:url" content="http://example.com/index.html">
<meta property="og:site_name" content="Sicilienne">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://raw.githubusercontent.com/Warmchay/img/main/1521647611449_.pic.jpg">
<meta property="article:author" content="Zoris">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://raw.githubusercontent.com/Warmchay/img/main/1521647611449_.pic.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="http://example.com/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@6/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = {
root: '/',
algolia: undefined,
localSearch: undefined,
translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"简","msgToSimplifiedChinese":"繁"},
noticeOutdate: undefined,
highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
copy: {
success: '复制成功',
error: '复制错误',
noSupport: '浏览器不支持'
},
relativeDate: {
homepage: false,
post: false
},
runtime: '',
date_suffix: {
just: '刚刚',
min: '分钟前',
hour: '小时前',
day: '天前',
month: '个月前'
},
copyright: undefined,
lightbox: 'fancybox',
Snackbar: undefined,
source: {
justifiedGallery: {
js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
}
},
isPhotoFigcaption: false,
islazyload: false,
isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
title: 'Sicilienne',
isPost: false,
isHome: true,
isHighlightShrink: false,
isToc: false,
postUpdate: '2022-05-20 12:43:14'
}</script><noscript><style type="text/css">
#nav {
opacity: 1
}
.justified-gallery img {
opacity: 1
}
#recent-posts time,
#post-meta time {
display: inline !important
}
</style></noscript><script>(win=>{
win.saveToLocal = {
set: function setWithExpiry(key, value, ttl) {
if (ttl === 0) return
const now = new Date()
const expiryDay = ttl * 86400000
const item = {
value: value,
expiry: now.getTime() + expiryDay,
}
localStorage.setItem(key, JSON.stringify(item))
},
get: function getWithExpiry(key) {
const itemStr = localStorage.getItem(key)
if (!itemStr) {
return undefined
}
const item = JSON.parse(itemStr)
const now = new Date()
if (now.getTime() > item.expiry) {
localStorage.removeItem(key)
return undefined
}
return item.value
}
}
win.getScript = url => new Promise((resolve, reject) => {
const script = document.createElement('script')
script.src = url
script.async = true
script.onerror = reject
script.onload = script.onreadystatechange = function() {
const loadState = this.readyState
if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
script.onload = script.onreadystatechange = null
resolve()
}
document.head.appendChild(script)
})
win.activateDarkMode = function () {
document.documentElement.setAttribute('data-theme', 'dark')
if (document.querySelector('meta[name="theme-color"]') !== null) {
document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
}
}
win.activateLightMode = function () {
document.documentElement.setAttribute('data-theme', 'light')
if (document.querySelector('meta[name="theme-color"]') !== null) {
document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
}
}
const t = saveToLocal.get('theme')
if (t === 'dark') activateDarkMode()
else if (t === 'light') activateLightMode()
const asideStatus = saveToLocal.get('aside-status')
if (asideStatus !== undefined) {
if (asideStatus === 'hide') {
document.documentElement.classList.add('hide-aside')
} else {
document.documentElement.classList.remove('hide-aside')
}
}
const detectApple = () => {
if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
document.documentElement.classList.add('apple')
}
}
detectApple()
})(window)</script><meta name="generator" content="Hexo 6.1.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="https://raw.githubusercontent.com/Warmchay/img/main/1521647611449_.pic.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">20</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">7</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">7</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 清单</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 电影</span></a></li><li><a class="site-page child" href="/Gallery/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="page" id="body-wrap"><header class="full_page" id="page-header" style="background-image: url('https://raw.githubusercontent.com/Warmchay/img/main/1561647612045_.pic.jpg')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">Sicilienne</a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 清单</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/music/"><i class="fa-fw fas fa-music"></i><span> 音乐</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> 电影</span></a></li><li><a class="site-page child" href="/Gallery/"><i class="fa-fw fas fa-images"></i><span> 照片</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="site-info"><h1 id="site-title">Sicilienne</h1><div id="site-subtitle"><span id="subtitle"></span></div><div id="site_social_icons"><a class="social-icon" href="https://github.com/Warmchay" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="mailto:[email protected]" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div><div id="scroll-down"><i class="fas fa-angle-down scroll-down-effects"></i></div></header><main class="layout" id="content-inner"><div class="recent-posts" id="recent-posts"><div class="recent-post-item"><div class="post_cover left"><a href="/2022/05/18/Distributed%20System/Hadoop%20%E5%8D%8F%E8%AE%AE%E5%A4%84%E7%90%86%E5%99%A8/" title="Hadoop 协议处理器"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop 协议处理器"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/05/18/Distributed%20System/Hadoop%20%E5%8D%8F%E8%AE%AE%E5%A4%84%E7%90%86%E5%99%A8/" title="Hadoop 协议处理器">Hadoop 协议处理器</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-05-18T01:55:15.000Z" title="发表于 2022-05-18 09:55:15">2022-05-18</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Distributed-System/">Distributed System</a></span></div><div class="content">
相关注释文件为:
org/apache/hadoop/fs/FsUrlStreamHandlerFactory.java
org/apache/hadoop/fs/FsUrlStreamHandler.java
org/apache/hadoop/fs/FsUrlConnection.java
1 引入Hadoop 中通过集成 FileSystem 抽象类，使其能在不同的文件系统中可移植，要从 Hadoop 文件系统中读取文件，最简单的方法是使用 java.net.URL 打开数据流，从中读数据。
无论是 JAVA 原生的 File 类还是 Hadoop 的 FileSystem 类，他们在将文件对象转换成对应 URI(Uniform Resource Identifier, 统一资源标识符) 时的方法都继承了 java.io.Serializable。所标识的资源可以是文件，也可以是电子邮件地址等。如绝对 URI 由 URI 模式和模式特有部分组成，它们之间由冒号隔开，常用的模式有: file, http, hftp, mailto, telnet, hdfs, har, s ...</div></div></div><div class="recent-post-item"><div class="post_cover right"><a href="/2022/05/17/Reading/LSM-Tree%20%E6%A6%82%E8%BF%B0/" title="LSM-Tree 概述"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LSM-Tree 概述"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/05/17/Reading/LSM-Tree%20%E6%A6%82%E8%BF%B0/" title="LSM-Tree 概述">LSM-Tree 概述</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-05-17T04:45:12.000Z" title="发表于 2022-05-17 12:45:12">2022-05-17</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Reading/">Reading</a></span></div><div class="content">摘要
相比于传统的 in-place updates，LSM-Tree 第一次写入都会缓存到内存中，通过后台的 flush 顺序追加到磁盘里，即 out-of-place update
LSM-Tree Basic回顾 LSM-tree 历史，讨论 LSM-Tree 结构，对 LSM-Tree 的写入/读取/空间利用作成本分析。
Historyindex-structure 分为就地更新和非就地更新两种。
图 a 直接覆盖旧记录，这种设计会牺牲写入性能，因为更新会导致随机 I/O，并且索引页通过更新和删除操作会产生分段，降低了空间利用率
为什么会导致随机 I/O？
图 b 将更新变化存储到新位置，不覆盖旧条目。这样设计利用顺序 I/O 来处理写操作，可以提高写性能；不覆盖旧数据，简化了恢复过程。
但是其存在的问题是：记录可能存储在任何位置，会牺牲读取性能。因此 out-of-place 结构通常需要单独的数据重组来提高存储/查询效率。
顺序进行非就地更新的想法在80年代已用于 log-structure ...</div></div></div><div class="recent-post-item"><div class="post_cover left"><a href="/2022/05/09/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%BA%8C)/" title="LevelDB 源码分析 (二)"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LevelDB 源码分析 (二)"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/05/09/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%BA%8C)/" title="LevelDB 源码分析 (二)">LevelDB 源码分析 (二)</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-05-09T01:55:15.000Z" title="发表于 2022-05-09 09:55:15">2022-05-09</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Distributed-System/">Distributed System</a></span></div><div class="content">内存管理 - Arena内存频繁创建和释放的地方一定会有内存池的出现，如在 levelDB 中 im/memtable 组件中，会有大量内存创建（数据的持续 put）和释放（dump到磁盘后释放内存）。其基本原理为:
levelDB 中内存管理的 component 为 Arena，其成员变量在 arena.h 中，log_format.h 中规定其块大小为 4K: kBlockSize=32768。
在实现 Arena (arena.cc)中，当内存剩余字节数小于所需分配时，根据所需分配字节是否大于 $kBlockSize/4$ 来确定是否需要使用当前块的剩余字节。
引用计数levelDB 中，memtable/version 等组件可能被多个线程共享，不能直接删除。而是需要等到无线程占用时，才能删除。
Key 和 Compare前置知识
SequenceNumber
递增的 uint64 整数，相同 key 按照其降序，最大值为:
123// We leave eight bits empty at the bottom so a type ...</div></div></div><div class="recent-post-item"><div class="post_cover right"><a href="/2022/05/07/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%B8%80)%20-%20%E5%89%8D%E6%B2%BF%E5%92%8C%E5%9F%BA%E6%9C%AC%E7%BB%84%E4%BB%B6/" title="LevelDB 源码分析 (一) ｜ 前沿和基本组件"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LevelDB 源码分析 (一) ｜ 前沿和基本组件"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/05/07/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%B8%80)%20-%20%E5%89%8D%E6%B2%BF%E5%92%8C%E5%9F%BA%E6%9C%AC%E7%BB%84%E4%BB%B6/" title="LevelDB 源码分析 (一) ｜ 前沿和基本组件">LevelDB 源码分析 (一) ｜ 前沿和基本组件</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-05-07T01:55:15.000Z" title="发表于 2022-05-07 09:55:15">2022-05-07</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Distributed-System/">Distributed System</a></span></div><div class="content">What is LevelDBLevelDB 是 KV 型数据库，为 BSD 协议，为 Leveling+分区 实现的 LSM 代表，适合写多读少。
特性
Key Value 支持任意 Byte 类型数组，不单单支持字符串
为持久化存储的 KV 系统，大部分数据存在磁盘上
按照 key 值顺序存储数据，按照用户定义的比较函数进行排序
操作接口简单，包括写/读记录以及删除记录，也支持针对多条操作的原子批量操作
支持 snapshot，读写操作互不影响
支持 snappy 压缩，I/O 效率高
源码编译
直接 clone
1git clone https://github.com/google/leveldb.git
initialize submodule issues#1004
12345# 一步到位git clone --recurse-submodules https://github.com/google/leveldb.git# without re-cloninggit submodule update --init --recursive
编译
...</div></div></div><div class="recent-post-item"><div class="post_cover left"><a href="/2022/05/04/Reading/Reading%20-%20%E5%AD%98%E5%82%A8%E4%B8%8A%E7%9A%84%E8%BD%AF%E7%A1%AC%E4%BB%B6%E5%8D%8F%E5%90%8C/" title="Reading | 存储上的软硬件协同"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Reading | 存储上的软硬件协同"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/05/04/Reading/Reading%20-%20%E5%AD%98%E5%82%A8%E4%B8%8A%E7%9A%84%E8%BD%AF%E7%A1%AC%E4%BB%B6%E5%8D%8F%E5%90%8C/" title="Reading | 存储上的软硬件协同">Reading | 存储上的软硬件协同</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-05-04T04:45:12.000Z" title="发表于 2022-05-04 12:45:12">2022-05-04</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Reading/">Reading</a></span></div><div class="content">存储上的软硬件协同Introduction
CPU 和 Memory 的 gap 越来越大，目前朝着多核发展有所缓和，但还是有很大的 gap
Disk 速度更慢，目前用一些软件方法来解决如操作系统将经常使用的数据存储在主存中，相比访问磁盘提高了访问速度
毫无疑问，主存是目前系统的瓶颈:
RAM 硬件设计 (速度和并行化)
内存控制做更优的设计
CPU caches
DMA for devices
现代硬件 (07年) - Commodity Hardware TodayIntro
除了 DDR4 还在发展，07 年的硬件和现在没有什么区别
最初的系统分为南北桥，北桥为 CPU 和内存控制器的接口，南桥主要为 I/O 的接口。
这种设计会有很多限制:
与 RAM 之间的通信都需要通过北桥
CPU 间的通信需要通过北桥上的同一总线
RAM 只有一个接口
改进设计: 将北桥与外部内存控制器作分离的设计，这样可以增大带宽
这样的设计瓶颈为北桥带宽
NUMA - 非一致性内存架构
CPU 相互通信也有一定成本
RAM Types分为 SRAM 和 ...</div></div></div><div class="recent-post-item"><div class="post_cover right"><a href="/2022/04/28/Reading/Reading%20-%20CDBTune/" title="Reading | CDBTune"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Reading | CDBTune"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/04/28/Reading/Reading%20-%20CDBTune/" title="Reading | CDBTune">Reading | CDBTune</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-04-28T09:22:12.000Z" title="发表于 2022-04-28 17:22:12">2022-04-28</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Reading/">Reading</a></span></div><div class="content">Reading - CDBTuneSummaryCDBTune 是一个帮助云数据库智能调优的工具，也是最先通过强化学习来找到调优策略方向的 Auto Tuning Tool。
CDBTune 的 contributions 主要为:
First end-to-end automatic database tuning system.
Use limited number of samples to learn the best knob settings by try-and-error manner in RL.
Design a new reward function in RL, accelerates the covergence speed.
Use DDPG(Deep Deterministic Policy Gradient) method to find the optimal configurations in high-dimensional continuous space.
Good adaptability and outperforms DBA and O ...</div></div></div><div class="recent-post-item"><div class="post_cover left"><a href="/2022/04/27/Reading/Reading%20-%20GFS/" title="Reading | Google File System"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Reading | Google File System"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/04/27/Reading/Reading%20-%20GFS/" title="Reading | Google File System">Reading | Google File System</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-04-27T00:20:12.000Z" title="发表于 2022-04-27 08:20:12">2022-04-27</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Reading/">Reading</a></span></div><div class="content">IntroGFS 设计前的观察 :
设备故障经常发生，GFS由许多台廉价设备作为存储节点组成，设备数量和质量决定任何时间都有部分设备无法工作，甚至有些设备会无法恢复。
文件越来越大。需要重新考虑 I/O 操作和 Chunk 大小的参数设计
大部分文件变更是以 Append 而非 Overwirte。
放宽一致性协议，能大幅简化系统，减少应用程序负担，
Design假设 Assumptions
系统由经常发生故障的廉价商用设备组成，而系统能持续监控自身并检测故障、容错和及时恢复的能力。
系统存储一定数量的大文件，但也能支持小文件
Workload 主要源于两种 Read 操作: 大规模的流式读取和小规模的随机读取。大规模中同一个 Client 端连续读操作会经常访问同一个区域，小规模中会在文件某个任意偏移位置读几KB
Workload 还来自很多文件的大规模 Append 写入。一般情况下当写入规模与读取规模相似时，文件一旦写入就不会被修改。
文件通常在 Producer-Consumer 队列中或 Multi-way Merge 中使用。因此最小化原子性需要的同步开销非常 ...</div></div></div><div class="recent-post-item"><div class="post_cover right"><a href="/2022/04/26/Reading/Reading%20-%20Ottertune/" title="Reading | OtterTune"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Reading | OtterTune"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/04/26/Reading/Reading%20-%20Ottertune/" title="Reading | OtterTune">Reading | OtterTune</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-04-26T05:40:12.000Z" title="发表于 2022-04-26 13:40:12">2022-04-26</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Reading/">Reading</a></span></div><div class="content">Ottertune 阅读Summary
Ottertune 为一个用于数据库的参数配置自动化工具，调优过程由三个步骤组成:
Workload Characterization ( Workload 特征采集/分析 ): 确定最能代表 target workload 的 metrics
Knobs affect metrics.
FA (factor analysis): 将 high-dimensional DBMS metric data 转为 low-dimensional data
K-means: Cluster thest lower dimensinal data into meaningful groups.将这些低维因子分组
**Knob Identification (重要参数分析): ** 使用 Lasso 方法确定 Knobs 的重要级排序
计算出哪些参数对性能影响最大及参数间的关系
Automatic Tuner (自动调优)
Step#1 - Workload Mapping: 根据已经挑选出的一组metrics，匹配 ta ...</div></div></div><div class="recent-post-item"><div class="post_cover left"><a href="/2022/04/25/Tools/How-to-read-a-paper/" title="How to read a paper?"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/1501647610811_.pic.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="How to read a paper?"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/04/25/Tools/How-to-read-a-paper/" title="How to read a paper?">How to read a paper?</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-04-25T14:12:12.000Z" title="发表于 2022-04-25 22:12:12">2022-04-25</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Tools/">Tools</a></span></div><div class="content">How to read感觉自己读论文不是很高效，一篇论文看一个白天，方法什么的在路上也很难全部说出来，此篇不断记录我是如何看论文的 (我应该如何看论文)
所有借鉴过的方法都会在引用给出，感兴趣可以点击康康。
Abstract
Why: purpose of this work
How: method employed
What: result they discovered
Conclusion: Meaningful?
Introduction
如果是小白，好好看
如果是老鸟，skip or read the first/last paragraphs.
Catch one’s attention.
Explain the research context.
From broad topic to sprcific issues to focused core questions.
Results and Discussion
What they found
Vital data, figures and tables
Must understand how t ...</div></div></div><div class="recent-post-item"><div class="post_cover right"><a href="/2022/04/23/Gossip/%E7%B3%BB%E7%BB%9F%E7%A0%94%E7%A9%B6%E7%9A%84%E4%B8%80%E7%82%B9%E6%80%9D%E8%80%83/" title="系统研究的一点思考"><img class="post_bg" src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.50.15.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="系统研究的一点思考"></a></div><div class="recent-post-info"><a class="article-title" href="/2022/04/23/Gossip/%E7%B3%BB%E7%BB%9F%E7%A0%94%E7%A9%B6%E7%9A%84%E4%B8%80%E7%82%B9%E6%80%9D%E8%80%83/" title="系统研究的一点思考">系统研究的一点思考</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2022-04-23T04:45:12.000Z" title="发表于 2022-04-23 12:45:12">2022-04-23</time></span><span class="article-meta"><span class="article-meta-separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/Gossip/">Gossip</a></span></div><div class="content">
不仅仅是科学
Computer System: Science + Engineering + Art
Science
Measure and quantitate existing phenomenon
Building models to understand the world
Engineering
Building useful systems and tools
做有用的工作
We build stuffs with flexibility
Reinvent a wheel
主要挑战: 复杂性
设计架构 + 实现
目标: STEADY开发: T-imely
性能: S-imple, E-fficient, A-daptable
功能: D-ependable, Y-ummy
容错: D-ependable
研究驱动力
工艺与硬件的变化
技术创新的进步
应用模式的变化
“埋头拉车” + “抬头看路”
埋头拉车: 在一个领域内埋头苦干, 深入钻研
Get hands dirty
Understand even the gory details
抬头 ...</div></div></div><nav id="pagination"><div class="pagination"><span class="page-number current">1</span><a class="page-number" href="/page/2/#content-inner">2</a><a class="extend next" rel="next" href="/page/2/#content-inner"><i class="fas fa-chevron-right fa-fw"></i></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="https://raw.githubusercontent.com/Warmchay/img/main/1521647611449_.pic.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">Zoris</div><div class="author-info__description"></div></div><div class="card-info-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">20</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">7</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">7</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/Warmchay"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/Warmchay" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="mailto:[email protected]" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">Come to your senses, defense is not the way to go.</div></div><div class="sticky_layout"><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2022/05/18/Distributed%20System/Hadoop%20%E5%8D%8F%E8%AE%AE%E5%A4%84%E7%90%86%E5%99%A8/" title="Hadoop 协议处理器"><img src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop 协议处理器"/></a><div class="content"><a class="title" href="/2022/05/18/Distributed%20System/Hadoop%20%E5%8D%8F%E8%AE%AE%E5%A4%84%E7%90%86%E5%99%A8/" title="Hadoop 协议处理器">Hadoop 协议处理器</a><time datetime="2022-05-18T01:55:15.000Z" title="发表于 2022-05-18 09:55:15">2022-05-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2022/05/17/Reading/LSM-Tree%20%E6%A6%82%E8%BF%B0/" title="LSM-Tree 概述"><img src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LSM-Tree 概述"/></a><div class="content"><a class="title" href="/2022/05/17/Reading/LSM-Tree%20%E6%A6%82%E8%BF%B0/" title="LSM-Tree 概述">LSM-Tree 概述</a><time datetime="2022-05-17T04:45:12.000Z" title="发表于 2022-05-17 12:45:12">2022-05-17</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2022/05/09/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%BA%8C)/" title="LevelDB 源码分析 (二)"><img src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LevelDB 源码分析 (二)"/></a><div class="content"><a class="title" href="/2022/05/09/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%BA%8C)/" title="LevelDB 源码分析 (二)">LevelDB 源码分析 (二)</a><time datetime="2022-05-09T01:55:15.000Z" title="发表于 2022-05-09 09:55:15">2022-05-09</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2022/05/07/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%B8%80)%20-%20%E5%89%8D%E6%B2%BF%E5%92%8C%E5%9F%BA%E6%9C%AC%E7%BB%84%E4%BB%B6/" title="LevelDB 源码分析 (一) ｜ 前沿和基本组件"><img src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-06%20%E4%B8%8A%E5%8D%889.52.45.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LevelDB 源码分析 (一) ｜ 前沿和基本组件"/></a><div class="content"><a class="title" href="/2022/05/07/Distributed%20System/LevelDB%20%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%20(%E4%B8%80)%20-%20%E5%89%8D%E6%B2%BF%E5%92%8C%E5%9F%BA%E6%9C%AC%E7%BB%84%E4%BB%B6/" title="LevelDB 源码分析 (一) ｜ 前沿和基本组件">LevelDB 源码分析 (一) ｜ 前沿和基本组件</a><time datetime="2022-05-07T01:55:15.000Z" title="发表于 2022-05-07 09:55:15">2022-05-07</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2022/05/04/Reading/Reading%20-%20%E5%AD%98%E5%82%A8%E4%B8%8A%E7%9A%84%E8%BD%AF%E7%A1%AC%E4%BB%B6%E5%8D%8F%E5%90%8C/" title="Reading | 存储上的软硬件协同"><img src="https://raw.githubusercontent.com/Warmchay/img/main/%E6%88%AA%E5%B1%8F2022-04-29%20%E4%B8%8B%E5%8D%8810.47.53.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Reading | 存储上的软硬件协同"/></a><div class="content"><a class="title" href="/2022/05/04/Reading/Reading%20-%20%E5%AD%98%E5%82%A8%E4%B8%8A%E7%9A%84%E8%BD%AF%E7%A1%AC%E4%BB%B6%E5%8D%8F%E5%90%8C/" title="Reading | 存储上的软硬件协同">Reading | 存储上的软硬件协同</a><time datetime="2022-05-04T04:45:12.000Z" title="发表于 2022-05-04 12:45:12">2022-05-04</time></div></div></div></div><div class="card-widget" id="card-newest-comments"><div class="item-headline"><i class="fas fa-comment-dots"></i><span>最新评论</span></div><div class="aside-list"><span>正在加载中...</span></div></div><div class="card-widget card-categories"><div class="item-headline">
<i class="fas fa-folder-open"></i>
<span>分类</span>
</div>
<ul class="card-category-list" id="aside-cat-list">
<li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Advanced-OS/"><span class="card-category-list-name">Advanced OS</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Distributed-System/"><span class="card-category-list-name">Distributed System</span><span class="card-category-list-count">4</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Gossip/"><span class="card-category-list-name">Gossip</span><span class="card-category-list-count">1</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Missing-Semester/"><span class="card-category-list-name">Missing Semester</span><span class="card-category-list-count">3</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Reading/"><span class="card-category-list-name">Reading</span><span class="card-category-list-count">5</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Static-Analysis/"><span class="card-category-list-name">Static Analysis</span><span class="card-category-list-count">3</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Tools/"><span class="card-category-list-name">Tools</span><span class="card-category-list-count">2</span></a></li>
</ul></div><div class="card-widget card-tags"><div class="item-headline"><i class="fas fa-tags"></i><span>标签</span></div><div class="card-tag-cloud"><a href="/tags/Clang/" style="font-size: 1.1em; color: #999">Clang</a> <a href="/tags/Database/" style="font-size: 1.5em; color: #99a9bf">Database</a> <a href="/tags/Memory/" style="font-size: 1.1em; color: #999">Memory</a> <a href="/tags/OS/" style="font-size: 1.2em; color: #999da3">OS</a> <a href="/tags/SA/" style="font-size: 1.3em; color: #99a1ac">SA</a> <a href="/tags/System/" style="font-size: 1.2em; color: #999da3">System</a> <a href="/tags/Tools/" style="font-size: 1.4em; color: #99a5b6">Tools</a></div></div><div class="card-widget card-archives"><div class="item-headline"><i class="fas fa-archive"></i><span>归档</span></div><ul class="card-archive-list"><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2022/05/"><span class="card-archive-list-date">五月 2022</span><span class="card-archive-list-count">5</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2022/04/"><span class="card-archive-list-date">四月 2022</span><span class="card-archive-list-count">9</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2022/03/"><span class="card-archive-list-date">三月 2022</span><span class="card-archive-list-count">6</span></a></li></ul></div><div class="card-widget card-webinfo"><div class="item-headline"><i class="fas fa-chart-line"></i><span>网站资讯</span></div><div class="webinfo"><div class="webinfo-item"><div class="item-name">文章数目 :</div><div class="item-count">20</div></div><div class="webinfo-item"><div class="item-name">本站访客数 :</div><div class="item-count" id="busuanzi_value_site_uv"></div></div><div class="webinfo-item"><div class="item-name">本站总访问量 :</div><div class="item-count" id="busuanzi_value_site_pv"></div></div><div class="webinfo-item"><div class="item-name">最后更新时间 :</div><div class="item-count" id="last-push-date" data-lastPushDate="2022-05-20T04:43:14.440Z"></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('https://raw.githubusercontent.com/Warmchay/img/main/1561647612045_.pic.jpg')"><div id="footer-wrap"><div class="copyright">©2022 By Zoris</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><div class="js-pjax"><script>function subtitleType () {
if (true) {
window.typed = new Typed("#subtitle", {
strings: ["Move on.","Never say never."],
startDelay: 300,
typeSpeed: 150,
loop: true,
backSpeed: 50
})
} else {
document.getElementById("subtitle").innerHTML = 'Move on.'
}
}
if (true) {
if (typeof Typed === 'function') {
subtitleType()
} else {
getScript('https://cdn.jsdelivr.net/npm/typed.js/lib/typed.min.js').then(subtitleType)
}
} else {
subtitleType()
}</script></div><div class="aplayer no-destroy" data-id="7319374102" data-server="netease" data-type="playlist" data-fixed="true" data-autoplay="true"> </div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/aplayer@1/dist/APlayer.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/aplayer@1/dist/APlayer.min.js"></script><script src="https://cdn.jsdelivr.net/gh/metowolf/[email protected]/dist/Meting.min.js"></script><script src="https://cdn.jsdelivr.net/npm/pjax/pjax.min.js"></script><script>let pjaxSelectors = ["title","#config-diff","#body-wrap","#rightside-config-hide","#rightside-config-show",".js-pjax"]
var pjax = new Pjax({
elements: 'a:not([target="_blank"])',
selectors: pjaxSelectors,
cacheBust: false,
analytics: false,
scrollRestoration: false
})
document.addEventListener('pjax:send', function () {
// removeEventListener scroll
window.tocScrollFn && window.removeEventListener('scroll', window.tocScrollFn)
window.scrollCollect && window.removeEventListener('scroll', scrollCollect)
typeof preloader === 'object' && preloader.initLoading()
document.getElementById('rightside').style.cssText = "opacity: ''; transform: ''"
if (window.aplayers) {
for (let i = 0; i < window.aplayers.length; i++) {
if (!window.aplayers[i].options.fixed) {
window.aplayers[i].destroy()
}
}
}
typeof typed === 'object' && typed.destroy()
//reset readmode
const $bodyClassList = document.body.classList
$bodyClassList.contains('read-mode') && $bodyClassList.remove('read-mode')
})
document.addEventListener('pjax:complete', function () {
window.refreshFn()
document.querySelectorAll('script[data-pjax]').forEach(item => {
const newScript = document.createElement('script')
const content = item.text || item.textContent || item.innerHTML || ""
Array.from(item.attributes).forEach(attr => newScript.setAttribute(attr.name, attr.value))
newScript.appendChild(document.createTextNode(content))
item.parentNode.replaceChild(newScript, item)
})
GLOBAL_CONFIG.islazyload && window.lazyLoadInstance.update()
typeof chatBtnFn === 'function' && chatBtnFn()
typeof panguInit === 'function' && panguInit()
// google analytics
typeof gtag === 'function' && gtag('config', '', {'page_path': window.location.pathname});
// baidu analytics
typeof _hmt === 'object' && _hmt.push(['_trackPageview',window.location.pathname]);
typeof loadMeting === 'function' && document.getElementsByClassName('aplayer').length && loadMeting()
// prismjs
typeof Prism === 'object' && Prism.highlightAll()
typeof preloader === 'object' && preloader.endLoading()
})
document.addEventListener('pjax:error', (e) => {
if (e.request.status === 404) {
pjax.loadUrl('/404.html')
}
})</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>