Skip to content

Commit 34a6800

Browse files
Andrei GudkovHuangSuiXiao
Andrei Gudkov
authored and
HuangSuiXiao
committed
migration/calc-dirty-rate: millisecond-granularity period
This patch allows to measure dirty page rate for sub-second intervals of time. An optional argument is introduced -- calc-time-unit. For example: {"execute": "calc-dirty-rate", "arguments": {"calc-time": 500, "calc-time-unit": "millisecond"} } Millisecond granularity allows to make predictions whether migration will succeed or not. To do this, calculate dirty rate with calc-time set to max allowed downtime (e.g. 300ms), convert measured rate into volume of dirtied memory, and divide by network throughput. If the value is lower than max allowed downtime, then migration will converge. Measurement results for single thread randomly writing to a 1/4/24GiB memory region: +----------------+-----------------------------------------------+ | calc-time | dirty rate MiB/s | | (milliseconds) +----------------+---------------+--------------+ | | theoretical | page-sampling | dirty-bitmap | | | (at 3M wr/sec) | | | +----------------+----------------+---------------+--------------+ | 1GiB | +----------------+----------------+---------------+--------------+ | 100 | 6996 | 7100 | 3192 | | 200 | 4606 | 4660 | 2655 | | 300 | 3305 | 3280 | 2371 | | 400 | 2534 | 2525 | 2154 | | 500 | 2041 | 2044 | 1871 | | 750 | 1365 | 1341 | 1358 | | 1000 | 1024 | 1052 | 1025 | | 1500 | 683 | 678 | 684 | | 2000 | 512 | 507 | 513 | +----------------+----------------+---------------+--------------+ | 4GiB | +----------------+----------------+---------------+--------------+ | 100 | 10232 | 8880 | 4070 | | 200 | 8954 | 8049 | 3195 | | 300 | 7889 | 7193 | 2881 | | 400 | 6996 | 6530 | 2700 | | 500 | 6245 | 5772 | 2312 | | 750 | 4829 | 4586 | 2465 | | 1000 | 3865 | 3780 | 2178 | | 1500 | 2694 | 2633 | 2004 | | 2000 | 2041 | 2031 | 1789 | +----------------+----------------+---------------+--------------+ | 24GiB | +----------------+----------------+---------------+--------------+ | 100 | 11495 | 8640 | 5597 | | 200 | 11226 | 8616 | 3527 | | 300 | 10965 | 8386 | 2355 | | 400 | 10713 | 8370 | 2179 | | 500 | 10469 | 8196 | 2098 | | 750 | 9890 | 7885 | 2556 | | 1000 | 9354 | 7506 | 2084 | | 1500 | 8397 | 6944 | 2075 | | 2000 | 7574 | 6402 | 2062 | +----------------+----------------+---------------+--------------+ Theoretical values are computed according to the following formula: size * (1 - (1-(4096/size))^(time*wps)) / (time * 2^20), where size is in bytes, time is in seconds, and wps is number of writes per second. Signed-off-by: Andrei Gudkov <[email protected]> Reviewed-by: Hyman Huang <[email protected]> Message-Id: <d802e6b8053eb60fbec1a784cf86f67d9528e0a8.1693895970.git.gudkov.andrei@huawei.com> Signed-off-by: Hyman Huang <[email protected]>
1 parent 2f3913f commit 34a6800

File tree

3 files changed

+128
-49
lines changed

3 files changed

+128
-49
lines changed

‎migration/dirtyrate.c

Lines changed: 74 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -189,10 +189,9 @@ int64_t vcpu_calculate_dirtyrate(int64_t calc_time_ms,
189189
return duration;
190190
}
191191

192-
static bool is_sample_period_valid(int64_t sec)
192+
static bool is_calc_time_valid(int64_t msec)
193193
{
194-
if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC ||
195-
sec > MAX_FETCH_DIRTYRATE_TIME_SEC) {
194+
if ((msec < MIN_CALC_TIME_MS) || (msec > MAX_CALC_TIME_MS)) {
196195
return false;
197196
}
198197

@@ -216,7 +215,39 @@ static int dirtyrate_set_state(int *state, int old_state, int new_state)
216215
}
217216
}
218217

219-
static struct DirtyRateInfo *query_dirty_rate_info(void)
218+
/* Decimal power of given time unit relative to one second */
219+
static int time_unit_to_power(TimeUnit time_unit)
220+
{
221+
switch (time_unit) {
222+
case TIME_UNIT_SECOND:
223+
return 0;
224+
case TIME_UNIT_MILLISECOND:
225+
return -3;
226+
default:
227+
assert(false); /* unreachable */
228+
return 0;
229+
}
230+
}
231+
232+
static int64_t convert_time_unit(int64_t value, TimeUnit unit_from,
233+
TimeUnit unit_to)
234+
{
235+
int power = time_unit_to_power(unit_from) -
236+
time_unit_to_power(unit_to);
237+
while (power < 0) {
238+
value /= 10;
239+
power += 1;
240+
}
241+
while (power > 0) {
242+
value *= 10;
243+
power -= 1;
244+
}
245+
return value;
246+
}
247+
248+
249+
static struct DirtyRateInfo *
250+
query_dirty_rate_info(TimeUnit calc_time_unit)
220251
{
221252
int i;
222253
int64_t dirty_rate = DirtyStat.dirty_rate;
@@ -225,7 +256,10 @@ static struct DirtyRateInfo *query_dirty_rate_info(void)
225256

226257
info->status = CalculatingState;
227258
info->start_time = DirtyStat.start_time;
228-
info->calc_time = DirtyStat.calc_time;
259+
info->calc_time = convert_time_unit(DirtyStat.calc_time_ms,
260+
TIME_UNIT_MILLISECOND,
261+
calc_time_unit);
262+
info->calc_time_unit = calc_time_unit;
229263
info->sample_pages = DirtyStat.sample_pages;
230264
info->mode = dirtyrate_mode;
231265

@@ -264,7 +298,7 @@ static void init_dirtyrate_stat(int64_t start_time,
264298
{
265299
DirtyStat.dirty_rate = -1;
266300
DirtyStat.start_time = start_time;
267-
DirtyStat.calc_time = config.sample_period_seconds;
301+
DirtyStat.calc_time_ms = config.calc_time_ms;
268302
DirtyStat.sample_pages = config.sample_pages_per_gigabytes;
269303

270304
switch (config.mode) {
@@ -574,7 +608,6 @@ static inline void dirtyrate_manual_reset_protect(void)
574608

575609
static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
576610
{
577-
int64_t msec = 0;
578611
int64_t start_time;
579612
DirtyPageRecord dirty_pages;
580613

@@ -602,9 +635,7 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
602635
start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
603636
DirtyStat.start_time = start_time / 1000;
604637

605-
msec = config.sample_period_seconds * 1000;
606-
msec = dirty_stat_wait(msec, start_time);
607-
DirtyStat.calc_time = msec / 1000;
638+
DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms, start_time);
608639

609640
/*
610641
* do two things.
@@ -615,12 +646,12 @@ static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config)
615646

616647
record_dirtypages_bitmap(&dirty_pages, false);
617648

618-
DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages, msec);
649+
DirtyStat.dirty_rate = do_calculate_dirtyrate(dirty_pages,
650+
DirtyStat.calc_time_ms);
619651
}
620652

621653
static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
622654
{
623-
int64_t duration;
624655
uint64_t dirtyrate = 0;
625656
uint64_t dirtyrate_sum = 0;
626657
int i = 0;
@@ -631,12 +662,10 @@ static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config)
631662
DirtyStat.start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000;
632663

633664
/* calculate vcpu dirtyrate */
634-
duration = vcpu_calculate_dirtyrate(config.sample_period_seconds * 1000,
635-
&DirtyStat.dirty_ring,
636-
GLOBAL_DIRTY_DIRTY_RATE,
637-
true);
638-
639-
DirtyStat.calc_time = duration / 1000;
665+
DirtyStat.calc_time_ms = vcpu_calculate_dirtyrate(config.calc_time_ms,
666+
&DirtyStat.dirty_ring,
667+
GLOBAL_DIRTY_DIRTY_RATE,
668+
true);
640669

641670
/* calculate vm dirtyrate */
642671
for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) {
@@ -652,7 +681,6 @@ static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
652681
{
653682
struct RamblockDirtyInfo *block_dinfo = NULL;
654683
int block_count = 0;
655-
int64_t msec = 0;
656684
int64_t initial_time;
657685

658686
rcu_read_lock();
@@ -662,17 +690,16 @@ static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config)
662690
}
663691
rcu_read_unlock();
664692

665-
msec = config.sample_period_seconds * 1000;
666-
msec = dirty_stat_wait(msec, initial_time);
693+
DirtyStat.calc_time_ms = dirty_stat_wait(config.calc_time_ms,
694+
initial_time);
667695
DirtyStat.start_time = initial_time / 1000;
668-
DirtyStat.calc_time = msec / 1000;
669696

670697
rcu_read_lock();
671698
if (!compare_page_hash_info(block_dinfo, block_count)) {
672699
goto out;
673700
}
674701

675-
update_dirtyrate(msec);
702+
update_dirtyrate(DirtyStat.calc_time_ms);
676703

677704
out:
678705
rcu_read_unlock();
@@ -718,6 +745,8 @@ void *get_dirtyrate_thread(void *arg)
718745
}
719746

720747
void qmp_calc_dirty_rate(int64_t calc_time,
748+
bool has_calc_time_unit,
749+
TimeUnit calc_time_unit,
721750
bool has_sample_pages,
722751
int64_t sample_pages,
723752
bool has_mode,
@@ -737,10 +766,15 @@ void qmp_calc_dirty_rate(int64_t calc_time,
737766
return;
738767
}
739768

740-
if (!is_sample_period_valid(calc_time)) {
741-
error_setg(errp, "calc-time is out of range[%d, %d].",
742-
MIN_FETCH_DIRTYRATE_TIME_SEC,
743-
MAX_FETCH_DIRTYRATE_TIME_SEC);
769+
int64_t calc_time_ms = convert_time_unit(
770+
calc_time,
771+
has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND,
772+
TIME_UNIT_MILLISECOND
773+
);
774+
775+
if (!is_calc_time_valid(calc_time_ms)) {
776+
error_setg(errp, "Calculation time is out of range [%dms, %dms].",
777+
MIN_CALC_TIME_MS, MAX_CALC_TIME_MS);
744778
return;
745779
}
746780

@@ -787,7 +821,7 @@ void qmp_calc_dirty_rate(int64_t calc_time,
787821
return;
788822
}
789823

790-
config.sample_period_seconds = calc_time;
824+
config.calc_time_ms = calc_time_ms;
791825
config.sample_pages_per_gigabytes = sample_pages;
792826
config.mode = mode;
793827

@@ -806,14 +840,18 @@ void qmp_calc_dirty_rate(int64_t calc_time,
806840
(void *)&config, QEMU_THREAD_DETACHED);
807841
}
808842

809-
struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp)
843+
844+
struct DirtyRateInfo *qmp_query_dirty_rate(bool has_calc_time_unit,
845+
TimeUnit calc_time_unit,
846+
Error **errp)
810847
{
811-
return query_dirty_rate_info();
848+
return query_dirty_rate_info(
849+
has_calc_time_unit ? calc_time_unit : TIME_UNIT_SECOND);
812850
}
813851

814852
void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict)
815853
{
816-
DirtyRateInfo *info = query_dirty_rate_info();
854+
DirtyRateInfo *info = query_dirty_rate_info(TIME_UNIT_SECOND);
817855

818856
monitor_printf(mon, "Status: %s\n",
819857
DirtyRateStatus_str(info->status));
@@ -873,8 +911,11 @@ void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict)
873911
mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING;
874912
}
875913

876-
qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true,
877-
mode, &err);
914+
qmp_calc_dirty_rate(sec, /* calc-time */
915+
false, TIME_UNIT_SECOND, /* calc-time-unit */
916+
has_sample_pages, sample_pages,
917+
true, mode,
918+
&err);
878919
if (err) {
879920
hmp_handle_error(mon, err);
880921
return;

‎migration/dirtyrate.h

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,12 @@
3131
#define MIN_RAMBLOCK_SIZE 128
3232

3333
/*
34-
* Take 1s as minimum time for calculation duration
34+
* Allowed range for dirty page rate calculation (in milliseconds).
35+
* Lower limit relates to the smallest realistic downtime it
36+
* makes sense to impose on migration.
3537
*/
36-
#define MIN_FETCH_DIRTYRATE_TIME_SEC 1
37-
#define MAX_FETCH_DIRTYRATE_TIME_SEC 60
38+
#define MIN_CALC_TIME_MS 50
39+
#define MAX_CALC_TIME_MS 60000
3840

3941
/*
4042
* Take 1/16 pages in 1G as the maxmum sample page count
@@ -44,7 +46,7 @@
4446

4547
struct DirtyRateConfig {
4648
uint64_t sample_pages_per_gigabytes; /* sample pages per GB */
47-
int64_t sample_period_seconds; /* time duration between two sampling */
49+
int64_t calc_time_ms; /* desired calculation time (in milliseconds) */
4850
DirtyRateMeasureMode mode; /* mode of dirtyrate measurement */
4951
};
5052

@@ -73,7 +75,7 @@ typedef struct SampleVMStat {
7375
struct DirtyRateStat {
7476
int64_t dirty_rate; /* dirty rate in MB/s */
7577
int64_t start_time; /* calculation start time in units of second */
76-
int64_t calc_time; /* time duration of two sampling in units of second */
78+
int64_t calc_time_ms; /* actual calculation time (in milliseconds) */
7779
uint64_t sample_pages; /* sample pages per GB */
7880
union {
7981
SampleVMStat page_sampling;

‎qapi/migration.json

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1836,6 +1836,21 @@
18361836
{ 'enum': 'DirtyRateMeasureMode',
18371837
'data': ['page-sampling', 'dirty-ring', 'dirty-bitmap'] }
18381838

1839+
##
1840+
# @TimeUnit:
1841+
#
1842+
# Specifies unit in which time-related value is specified.
1843+
#
1844+
# @second: value is in seconds
1845+
#
1846+
# @millisecond: value is in milliseconds
1847+
#
1848+
# Since 8.2
1849+
#
1850+
##
1851+
{ 'enum': 'TimeUnit',
1852+
'data': ['second', 'millisecond'] }
1853+
18391854
##
18401855
# @DirtyRateInfo:
18411856
#
@@ -1848,8 +1863,10 @@
18481863
#
18491864
# @start-time: start time in units of second for calculation
18501865
#
1851-
# @calc-time: time period for which dirty page rate was measured
1852-
# (in seconds)
1866+
# @calc-time: time period for which dirty page rate was measured,
1867+
# expressed and rounded down to @calc-time-unit.
1868+
#
1869+
# @calc-time-unit: time unit of @calc-time (Since 8.2)
18531870
#
18541871
# @sample-pages: number of sampled pages per GiB of guest memory.
18551872
# Valid only in page-sampling mode (Since 6.1)
@@ -1866,6 +1883,7 @@
18661883
'status': 'DirtyRateStatus',
18671884
'start-time': 'int64',
18681885
'calc-time': 'int64',
1886+
'calc-time-unit': 'TimeUnit',
18691887
'sample-pages': 'uint64',
18701888
'mode': 'DirtyRateMeasureMode',
18711889
'*vcpu-dirty-rate': [ 'DirtyRateVcpu' ] } }
@@ -1901,12 +1919,16 @@
19011919
# This mode tracks page modification per each vCPU separately. It
19021920
# requires that KVM accelerator property "dirty-ring-size" is set.
19031921
#
1904-
# @calc-time: time period in units of second for which dirty page rate
1905-
# is calculated. Note that larger @calc-time values will
1906-
# typically result in smaller dirty page rates because page
1907-
# dirtying is a one-time event. Once some page is counted as
1908-
# dirty during @calc-time period, further writes to this page will
1909-
# not increase dirty page rate anymore.
1922+
# @calc-time: time period for which dirty page rate is calculated.
1923+
# By default it is specified in seconds, but the unit can be set
1924+
# explicitly with @calc-time-unit. Note that larger @calc-time
1925+
# values will typically result in smaller dirty page rates because
1926+
# page dirtying is a one-time event. Once some page is counted
1927+
# as dirty during @calc-time period, further writes to this page
1928+
# will not increase dirty page rate anymore.
1929+
#
1930+
# @calc-time-unit: time unit in which @calc-time is specified.
1931+
# By default it is seconds. (Since 8.2)
19101932
#
19111933
# @sample-pages: number of sampled pages per each GiB of guest memory.
19121934
# Default value is 512. For 4KiB guest pages this corresponds to
@@ -1924,8 +1946,16 @@
19241946
# -> {"execute": "calc-dirty-rate", "arguments": {"calc-time": 1,
19251947
# 'sample-pages': 512} }
19261948
# <- { "return": {} }
1949+
#
1950+
# Measure dirty rate using dirty bitmap for 500 milliseconds:
1951+
#
1952+
# -> {"execute": "calc-dirty-rate", "arguments": {"calc-time": 500,
1953+
# "calc-time-unit": "millisecond", "mode": "dirty-bitmap"} }
1954+
#
1955+
# <- { "return": {} }
19271956
##
19281957
{ 'command': 'calc-dirty-rate', 'data': {'calc-time': 'int64',
1958+
'*calc-time-unit': 'TimeUnit',
19291959
'*sample-pages': 'int',
19301960
'*mode': 'DirtyRateMeasureMode'} }
19311961

@@ -1934,21 +1964,27 @@
19341964
#
19351965
# Query results of the most recent invocation of @calc-dirty-rate.
19361966
#
1967+
# @calc-time-unit: time unit in which to report calculation time.
1968+
# By default it is reported in seconds. (Since 8.2)
1969+
#
19371970
# Since: 5.2
19381971
#
19391972
# Examples:
19401973
#
19411974
# 1. Measurement is in progress:
19421975
#
19431976
# <- {"status": "measuring", "sample-pages": 512,
1944-
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10}
1977+
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10,
1978+
# "calc-time-unit": "second"}
19451979
#
19461980
# 2. Measurement has been completed:
19471981
#
19481982
# <- {"status": "measured", "sample-pages": 512, "dirty-rate": 108,
1949-
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10}
1983+
# "mode": "page-sampling", "start-time": 3665220, "calc-time": 10,
1984+
# "calc-time-unit": "second"}
19501985
##
1951-
{ 'command': 'query-dirty-rate', 'returns': 'DirtyRateInfo' }
1986+
{ 'command': 'query-dirty-rate', 'data': {'*calc-time-unit': 'TimeUnit' },
1987+
'returns': 'DirtyRateInfo' }
19521988

19531989
##
19541990
# @DirtyLimitInfo:

0 commit comments

Comments
 (0)