forked from NAUbackup/VmBackup
-
Notifications
You must be signed in to change notification settings - Fork 0
/
VmBackup.py
1675 lines (1467 loc) · 66 KB
/
VmBackup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/python
#
#NAUVmBackup/VmBackup.py
# V3.25 August 2019
#
#@NAUbackup - NAU/ITS Department:
# Douglas Pace
# David McArthur
# Duane Booher
# Tobias Kreidl
#
# With external contributions gratefully made by:
# @philippmk -
# @ilium007 -
# @HqWisen -
# @JHag6694 -
# @lancefogle - Lance Fogle
# Tom McKelvey
# Copyright (C) 2019 Northern Arizona University
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Title: NAUbackup/VmBackup - a XenServer vm-export & vdi-export Backup Script
# Package Contents: README.md, VmBackup.py (this file), example.cfg
# Version History
# - v3.25 2019/08/07 Reconcile XenAPI.Session to be compatible
# with XenServer 6.X - 8.X releases, alert users in README file that
# session.xenapi.VM.get_by_name_label also returns name_labels
# of templates and hence should be avoided for VMs.
# - v3.24 2019/04/19 Fix lingering duplicate VM issues
# - v3.23 2018/06/13 Add preview check and execution check for duplicate VM
# names (potentially conflicting with snapshots),
# Add pre_clean option to delete oldest backups beforehand,
# fix subtle bug in pre-removing non-existing VMs from exclude list,
# add hostname to email subject line
# - v3.22 2017/11/11 Add full VM metadata dump to XML file to replace VM
# metadata backup that could fail if special characters encountered
# Added name_description UNICODE fix. (2018-Mar-20)
# Fixed bug in global definitions for vdi-export case. (2018-Mar-20)
# - v3.21 2017/09/29 Fix "except socket.error" syntax to also work with older
# python version in XenServer 6.X
# - v3.2 2017/09/12 Fix wildcard handling and excludes for both VM and VDI
# cases, add email retries
# - v3.1 2016/11/26 Added regexp include/exclude syntax for selecting VMs,
# checking writability of backup directory, SMTP TLS email option,
# define DEFAULT_STATUS_LOG parameter
# - v3.0 2016/01/20 Added vdi-export for VMs with too many/large disks for
# vm-export
# - v2.1 2014/08/22 Added email status option
# - v2.0 2014/04/09 New VmBackup version (supersedes all previous NAUbackup
# versions)
# ** DO NOT RUN THIS SCRIPT UNLESS YOU ARE COMFORTABLE WITH THESE ACTIONS. **
# => To accomplish the vm backup this script uses the following xe commands
# vm-export: (a) vm-snapshot, (b) template-param-set, (c) vm-export, (d) vm-uninstall on vm-snapshot
# vdi-export: (a) vdi-snapshot, (b) vdi-param-set, (c) vdi-export, (d) vdi-destroy on vdi-snapshot
# See README for usage and installation documentation.
# See example.cfg for config file example usage.
# Usage w/ vm name for single vm backup, which runs vm-export by default:
# ./VmBackup.py <password> <vm-name>
# Usage w/ config file for multiple vm backups, where you can specify either vm-export or vdi-export:
# ./VmBackup.py <password> <config-file-path>
import sys, time, os, datetime, subprocess, re, shutil, XenAPI, smtplib, re, base64, socket, threading, ssl
from email.MIMEText import MIMEText
from subprocess import PIPE
from subprocess import STDOUT
from os.path import join
############################# HARD CODED DEFAULTS
# modify these hard coded default values, only used if not specified in config file
DEFAULT_POOL_DB_BACKUP = 0
DEFAULT_MAX_BACKUPS = 4
DEFAULT_VDI_EXPORT_FORMAT = 'raw' # xe vdi-export options: 'raw' or 'vhd'
DEFAULT_BACKUP_DIR = '/snapshots/BACKUPS'
## DEFAULT_BACKUP_DIR = '\snapshots\BACKUPS' # alt for CIFS mounts
# note - some NAS file servers may fail with ':', so change to your desired format
BACKUP_DIR_PATTERN = '%s/backup-%04d-%02d-%02d-(%02d:%02d:%02d)'
DEFAULT_STATUS_LOG = '/snapshots/NAUbackup/status.log'
############################# OPTIONAL
# optional email may be triggered by configure next 3 parameters then find MAIL_ENABLE and uncommenting out the desired lines
MAIL_TO_ADDR = 'your-email@your-domain'
# note if MAIL_TO_ADDR has ipaddr then you may need to change the smtplib.SMTP() call
MAIL_FROM_ADDR = 'your-from-address@your-domain'
MAIL_SMTP_SERVER = 'your-mail-server'
config = {}
all_vms = []
expected_keys = ['pool_db_backup', 'max_backups', 'backup_dir', 'status_log', 'vdi_export_format', 'vm-export', 'vdi-export', 'exclude']
message = ''
xe_path = '/opt/xensource/bin'
def main(session):
success_cnt = 0
warning_cnt = 0
error_cnt = 0
#setting autoflush on (aka unbuffered)
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
server_name = os.uname()[1].split('.')[0]
if config_specified:
status_log_begin(server_name)
log('===========================')
log('VmBackup running on %s ...' % server_name)
log('===========================')
log('Check if backup directory %s is writable ...' % config['backup_dir'])
touchfile = os.path.join(config['backup_dir'], "00VMbackupWriteTest")
cmd = '/bin/touch "%s"' % touchfile
log(cmd)
res = run(cmd)
if not res:
log('ERROR failed to write to backup directory area - FATAL ERROR')
sys.exit(1)
else:
cmd = '/bin/rm -f "%s"' % touchfile
res = run(cmd)
log('Success: backup directory area is writable')
log('===========================')
df_snapshots('Space before backups: df -Th %s' % config['backup_dir'])
if int(config['pool_db_backup']):
log('*** begin backup_pool_metadata ***')
if not backup_pool_metadata(server_name):
error_cnt += 1
######################################################################
# Iterate through all vdi-export= in cfg
log('************ vdi-export= ***************')
for vm_parm in config['vdi-export']:
log('*** vdi-export begin %s' % vm_parm)
beginTime = datetime.datetime.now()
this_status = 'success'
# get values from vdi-export=
vm_name = get_vm_name(vm_parm)
vm_max_backups = get_vm_max_backups(vm_parm)
log('vdi-export - vm_name: %s max_backups: %s' % (vm_name, vm_max_backups))
if config_specified:
status_log_vdi_export_begin(server_name, '%s' % vm_name)
# verify vm_name exists with only one instance for this name
# returns error-message or vm_object if success
vm_object = verify_vm_name(vm_name)
if 'ERROR' in vm_object:
log('verify_vm_name: %s' % vm_object)
if config_specified:
status_log_vdi_export_end(server_name, 'ERROR verify_vm_name %s' % vm_name)
error_cnt += 1
# next vm
continue
vm_backup_dir = os.path.join(config['backup_dir'], vm_name)
# cleanup any old unsuccessful backups and create new full_backup_dir
full_backup_dir = process_backup_dir(vm_backup_dir)
# gather_vm_meta produces status: empty or warning-message
# and globals: vm_uuid, xvda_uuid, xvda_uuid
# => now only need: vm_uuid
# since all VM metadta go into an XML file
vm_meta_status = gather_vm_meta(vm_object, full_backup_dir)
if vm_meta_status != '':
log('WARNING gather_vm_meta: %s' % vm_meta_status)
this_status = 'warning'
# non-fatal - finsh processing for this vm
# vdi-export only uses xvda_uuid, xvda_uuid
if xvda_uuid == '':
log('ERROR gather_vm_meta has no xvda-uuid')
if config_specified:
status_log_vdi_export_end(server_name, 'ERROR xvda-uuid not found %s' % vm_name)
error_cnt += 1
# next vm
continue
if xvda_name_label == '':
log('ERROR gather_vm_meta has no xvda-name-label')
if config_specified:
status_log_vdi_export_end(server_name, 'ERROR xvda-name-label not found %s' % vm_name)
error_cnt += 1
# next vm
continue
# -----------------------------------------
# --- begin vdi-export command sequence ---
log ('*** vdi-export begin xe command sequence')
# is vm currently running?
cmd = '%s/xe vm-list name-label="%s" params=power-state | /bin/grep running' % (xe_path, vm_name)
if run_log_out_wait_rc(cmd) == 0:
log ('vm is running')
else:
log ('vm is NOT running')
# list the vdi we will backup
cmd = '%s/xe vdi-list uuid=%s' % (xe_path, xvda_uuid)
log('1.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('ERROR %s' % cmd)
if config_specified:
status_log_vdi_export_end(server_name, 'VDI-LIST-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# check for old vdi-snapshot for this xvda
snap_vdi_name_label = 'SNAP_%s_%s' % (vm_name, xvda_name_label)
# replace all spaces with '-'
snap_vdi_name_label = re.sub(r' ', r'-', snap_vdi_name_label)
log ('check for prev-vdi-snapshot: %s' % snap_vdi_name_label)
cmd = "%s/xe vdi-list name-label='%s' params=uuid | /bin/awk -F': ' '{print $2}' | /bin/grep '-'" % (xe_path, snap_vdi_name_label)
old_snap_vdi_uuid = run_get_lastline(cmd)
if old_snap_vdi_uuid != '':
log ('cleanup old-snap-vdi-uuid: %s' % old_snap_vdi_uuid)
# vdi-destroy old vdi-snapshot
cmd = '%s/xe vdi-destroy uuid=%s' % (xe_path, old_snap_vdi_uuid)
log('cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('WARNING %s' % cmd)
this_status = 'warning'
# non-fatal - finish processing for this vm
# === pre_cleanup code goes in here ===
if pre_clean:
pre_cleanup ( vm_backup_dir, vm_max_backups)
# take a vdi-snapshot of this vm
cmd = '%s/xe vdi-snapshot uuid=%s' % (xe_path, xvda_uuid)
log('2.cmd: %s' % cmd)
snap_vdi_uuid = run_get_lastline(cmd)
log ('snap-uuid: %s' % snap_vdi_uuid)
if snap_vdi_uuid == '':
log('ERROR %s' % cmd)
if config_specified:
status_log_vdi_export_end(server_name, 'VDI-SNAPSHOT-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# change vdi-snapshot to unique name-label for easy id and cleanup
cmd = '%s/xe vdi-param-set uuid=%s name-label="%s"' % (xe_path, snap_vdi_uuid, snap_vdi_name_label)
log('3.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('ERROR %s' % cmd)
if config_specified:
status_log_vdi_export_end(server_name, 'VDI-PARAM-SET-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# actual-backup: vdi-export vdi-snapshot
cmd = '%s/xe vdi-export format=%s uuid=%s' % (xe_path, config['vdi_export_format'], snap_vdi_uuid)
full_path_backup_file = os.path.join(full_backup_dir, vm_name + '.%s' % config['vdi_export_format'])
cmd = '%s filename="%s"' % (cmd, full_path_backup_file)
log('4.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) == 0:
log('vdi-export success')
else:
log('ERROR %s' % cmd)
if config_specified:
status_log_vdi_export_end(server_name, 'VDI-EXPORT-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# cleanup: vdi-destroy vdi-snapshot
cmd = '%s/xe vdi-destroy uuid=%s' % (xe_path, snap_vdi_uuid)
log('5.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('WARNING %s' % cmd)
this_status = 'warning'
# non-fatal - finsh processing for this vm
log ('*** vdi-export end')
# --- end vdi-export command sequence ---
# ---------------------------------------
elapseTime = datetime.datetime.now() - beginTime
backup_file_size = os.path.getsize(full_path_backup_file) / (1024 * 1024 * 1024)
final_cleanup( full_path_backup_file, backup_file_size, full_backup_dir, vm_backup_dir, vm_max_backups)
if not check_all_backups_success(vm_backup_dir):
log('WARNING cleanup needed - not all backup history is successful')
this_status = 'warning'
if (this_status == 'success'):
success_cnt += 1
log('VmBackup vdi-export %s - ***Success*** t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vdi_export_end(server_name, 'SUCCESS %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
elif (this_status == 'warning'):
warning_cnt += 1
log('VmBackup vdi-export %s - ***WARNING*** t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vdi_export_end(server_name, 'WARNING %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
else:
# this should never occur since all errors do a continue on to the next vm_name
error_cnt += 1
log('VmBackup vdi-export %s - +++ERROR-INTERNAL+++ t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vdi_export_end(server_name, 'ERROR-INTERNAL %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
# end of for vm_parm in config['vdi-export']:
######################################################################
######################################################################
# Iterate through all vm-export= in cfg
log('************ vm-export= ***************')
for vm_parm in config['vm-export']:
log('*** vm-export begin %s' % vm_parm)
beginTime = datetime.datetime.now()
this_status = 'success'
# get values from vdi-export=
vm_name = get_vm_name(vm_parm)
vm_max_backups = get_vm_max_backups(vm_parm)
log('vm-export - vm_name: %s max_backups: %s' % (vm_name, vm_max_backups))
if config_specified:
status_log_vm_export_begin(server_name, '%s' % vm_name)
vm_object = verify_vm_name(vm_name)
if 'ERROR' in vm_object:
log('verify_vm_name: %s' % vm_object)
if config_specified:
status_log_vm_export_end(server_name, 'ERROR verify_vm_name %s' % vm_name)
error_cnt += 1
# next vm
continue
vm_backup_dir = os.path.join(config['backup_dir'], vm_name)
# cleanup any old unsuccessful backups and create new full_backup_dir
full_backup_dir = process_backup_dir(vm_backup_dir)
# gather_vm_meta produces status: empty or warning-message
# and globals: vm_uuid, xvda_uuid, xvda_uuid
vm_meta_status = gather_vm_meta(vm_object, full_backup_dir)
if vm_meta_status != '':
log('WARNING gather_vm_meta: %s' % vm_meta_status)
this_status = 'warning'
# non-fatal - finsh processing for this vm
# vm-export only uses vm_uuid
if vm_uuid == '':
log('ERROR gather_vm_meta has no vm-uuid')
if config_specified:
status_log_vm_export_end(server_name, 'ERROR vm-uuid not found %s' % vm_name)
error_cnt += 1
# next vm
continue
# ----------------------------------------
# --- begin vm-export command sequence ---
log ('*** vm-export begin xe command sequence')
# is vm currently running?
cmd = '%s/xe vm-list name-label="%s" params=power-state | /bin/grep running' % (xe_path, vm_name)
if run_log_out_wait_rc(cmd) == 0:
log ('vm is running')
else:
log ('vm is NOT running')
# check for old vm-snapshot for this vm
snap_name = 'RESTORE_%s' % vm_name
log ('check for prev-vm-snapshot: %s' % snap_name)
cmd = "%s/xe vm-list name-label='%s' params=uuid | /bin/awk -F': ' '{print $2}' | /bin/grep '-'" % (xe_path, snap_name)
old_snap_vm_uuid = run_get_lastline(cmd)
if old_snap_vm_uuid != '':
log ('cleanup old-snap-vm-uuid: %s' % old_snap_vm_uuid)
# vm-uninstall old vm-snapshot
cmd = '%s/xe vm-uninstall uuid=%s force=true' % (xe_path, old_snap_vm_uuid)
log('cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('WARNING-ERROR %s' % cmd)
this_status = 'warning'
if config_specified:
status_log_vm_export_end(server_name, 'VM-UNINSTALL-FAIL-1 %s' % vm_name)
# non-fatal - finsh processing for this vm
# === pre_cleanup code goes in here ===
#print 'vm_backup_dir: %s' % vm_backup_dir
#print 'vm_max_backups: %s' % vm_max_backups
if pre_clean:
pre_cleanup (vm_backup_dir, vm_max_backups)
# take a vm-snapshot of this vm
cmd = '%s/xe vm-snapshot vm=%s new-name-label="%s"' % (xe_path, vm_uuid, snap_name)
log('1.cmd: %s' % cmd)
snap_vm_uuid = run_get_lastline(cmd)
log ('snap-uuid: %s' % snap_vm_uuid)
if snap_vm_uuid == '':
log('ERROR %s' % cmd)
if config_specified:
status_log_vm_export_end(server_name, 'SNAPSHOT-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# change vm-snapshot so that it can be referenced by vm-export
cmd = '%s/xe template-param-set is-a-template=false ha-always-run=false uuid=%s' % (xe_path, snap_vm_uuid)
log('2.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('ERROR %s' % cmd)
if config_specified:
status_log_vm_export_end(server_name, 'TEMPLATE-PARAM-SET-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# vm-export vm-snapshot
cmd = '%s/xe vm-export uuid=%s' % (xe_path, snap_vm_uuid)
if compress:
full_path_backup_file = os.path.join(full_backup_dir, vm_name + '.xva.gz')
cmd = '%s filename="%s" compress=true' % (cmd, full_path_backup_file)
else:
full_path_backup_file = os.path.join(full_backup_dir, vm_name + '.xva')
cmd = '%s filename="%s"' % (cmd, full_path_backup_file)
log('3.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) == 0:
log('vm-export success')
else:
log('ERROR %s' % cmd)
if config_specified:
status_log_vm_export_end(server_name, 'VM-EXPORT-FAIL %s' % vm_name)
error_cnt += 1
# next vm
continue
# vm-uninstall vm-snapshot
cmd = '%s/xe vm-uninstall uuid=%s force=true' % (xe_path, snap_vm_uuid)
log('4.cmd: %s' % cmd)
if run_log_out_wait_rc(cmd) != 0:
log('WARNING %s' % cmd)
this_status = 'warning'
# non-fatal - finsh processing for this vm
log ('*** vm-export end')
# --- end vm-export command sequence ---
# ----------------------------------------
elapseTime = datetime.datetime.now() - beginTime
backup_file_size = os.path.getsize(full_path_backup_file) / (1024 * 1024 * 1024)
final_cleanup( full_path_backup_file, backup_file_size, full_backup_dir, vm_backup_dir, vm_max_backups)
if not check_all_backups_success(vm_backup_dir):
log('WARNING cleanup needed - not all backup history is successful')
this_status = 'warning'
if (this_status == 'success'):
success_cnt += 1
log('VmBackup vm-export %s - ***Success*** t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vm_export_end(server_name, 'SUCCESS %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
elif (this_status == 'warning'):
warning_cnt += 1
log('VmBackup vm-export %s - ***WARNING*** t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vm_export_end(server_name, 'WARNING %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
else:
# this should never occur since all errors do a continue on to the next vm_name
error_cnt += 1
log('VmBackup vm-export %s - +++ERROR-INTERNAL+++ t:%s' % (vm_name, str(elapseTime.seconds/60)))
if config_specified:
status_log_vm_export_end(server_name, 'ERROR-INTERNAL %s,elapse:%s size:%sG' % (vm_name, str(elapseTime.seconds/60), backup_file_size))
# end of for vm_parm in config['vm-export']:
######################################################################
log('===========================')
df_snapshots('Space status: df -Th %s' % config['backup_dir'])
# gather a final VmBackup.py status
summary = 'S:%s W:%s E:%s' % (success_cnt, warning_cnt, error_cnt)
status_log = config['status_log']
if (error_cnt > 0):
if config_specified:
status_log_end(server_name, 'ERROR,%s' % summary)
# MAIL_ENABLE: optional email may be enabled by uncommenting out the next two lines
#send_email(MAIL_TO_ADDR, 'ERROR ' + os.uname()[1] + ' VmBackup.py', status_log)
#open('%s' % status_log, 'w').close() # trunc status log after email
log('VmBackup ended - **ERRORS DETECTED** - %s' % summary)
elif (warning_cnt > 0):
if config_specified:
status_log_end(server_name, 'WARNING,%s' % summary)
# MAIL_ENABLE: optional email may be enabled by uncommenting out the next two lines
#send_email(MAIL_TO_ADDR,'WARNING ' + os.uname()[1] + ' VmBackup.py', status_log)
#open('%s' % status_log, 'w').close() # trunc status log after email
log('VmBackup ended - **WARNING(s)** - %s' % summary)
else:
if config_specified:
status_log_end(server_name, 'SUCCESS,%s' % summary)
# MAIL_ENABLE: optional email may be enabled by uncommenting out the next two lines
#send_email(MAIL_TO_ADDR, 'Success ' + os.uname()[1] + ' VmBackup.py', status_log)
#open('%s' % status_log, 'w').close() # trunc status log after email
log('VmBackup ended - Success - %s' % summary)
# done with main()
######################################################################
def isInt(s):
try:
int(s)
return True
except ValueError:
return False
def get_vm_max_backups(vm_parm):
# get max_backups from optional vm-export=VM-NAME:MAX-BACKUP override
# NOTE - if not present then return config['max_backups']
if vm_parm.find(':') == -1:
return int(config['max_backups'])
else:
(vm_name,tmp_max_backups) = vm_parm.split(':')
tmp_max_backups = int(tmp_max_backups)
if (tmp_max_backups > 0):
return tmp_max_backups
else:
return int(config['max_backups'])
def is_vm_backups_valid(vm_parm):
if vm_parm.find(':') == -1:
# valid since we will use config['max_backups']
return True
else:
# a value has been specified - is it valid?
(vm_name,tmp_max_backups) = vm_parm.split(':')
if isInt(tmp_max_backups):
return tmp_max_backups > 0
else:
return False
def get_vm_backups(vm_parm):
# get max_backups from optional vm-export=VM-NAME:MAX-BACKUP override
# NOTE - if not present then return empty string '' else return whatever specified after ':'
if vm_parm.find(':') == -1:
return ''
else:
(vm_name,tmp_max_backups) = vm_parm.split(':')
return tmp_max_backups
def get_vm_name(vm_parm):
# get vm_name from optional vm-export=VM-NAME:MAX-BACKUP override
if (vm_parm.find(':') == -1):
return vm_parm
else:
(tmp_vm_name,tmp_max_backups) = vm_parm.split(':')
return tmp_vm_name
def verify_vm_name(tmp_vm_name):
vm = session.xenapi.VM.get_by_name_label(tmp_vm_name)
vmref = [x for x in session.xenapi.VM.get_by_name_label(tmp_vm_name) if not session.xenapi.VM.get_is_a_snapshot(x)]
if (len(vmref) > 1):
log ("ERROR: duplicate VM name found: %s | %s" % (tmp_vm_name, vmref))
return 'ERROR more than one vm with the name %s' % tmp_vm_name
elif (len(vm) == 0):
return 'ERROR no machines found with the name %s' % tmp_vm_name
return vm[0]
def gather_vm_meta(vm_object, tmp_full_backup_dir):
global vm_uuid
global xvda_uuid
global xvda_name_label
vm_uuid = ''
xvda_uuid = ''
xvda_name_label = ''
tmp_error = '';
vm_record = session.xenapi.VM.get_record(vm_object)
vm_uuid = vm_record['uuid']
log ('Exporting VM metadata XML info')
cmd = '%s/xe vm-export metadata=true uuid=%s filename= | tar -xOf - | /usr/bin/xmllint -format - > "%s/vm-metadata.xml"' % (xe_path, vm_uuid, tmp_full_backup_dir)
if run_log_out_wait_rc(cmd) != 0:
log('WARNING %s' % cmd)
this_status = 'warning'
# non-fatal - finish processing for this vm
log ('*** vm-export metadata end')
### The backup of the VM metadata portion in the code section below is
### deprecated since some entries such as name_label can contain
### non-standard characters that result in errors. All metadata are now saved
### using the code above. The additional VIF, Disk, VDI and VBD outputs
### are retained for now.
# # Backup vm meta data
# log ('Writing vm config file.')
# vm_out = open ('%s/vm.cfg' % tmp_full_backup_dir, 'w')
# vm_out.write('name_label=%s\n' % vm_record['name_label'])
# vm_out.write('name_description=%s\n' % vm_record['name_description'])
# vm_out.write('memory_dynamic_max=%s\n' % vm_record['memory_dynamic_max'])
# vm_out.write('VCPUs_max=%s\n' % vm_record['VCPUs_max'])
# vm_out.write('VCPUs_at_startup=%s\n' % vm_record['VCPUs_at_startup'])
# # notice some keys are not always available
# try:
# # notice list within list : vm_record['other_config']['base_template_name']
# vm_out.write('base_template_name=%s\n' % vm_record['other_config']['base_template_name'])
# except KeyError:
# # ignore
# pass
# vm_out.write('os_version=%s\n' % get_os_version(vm_record['uuid']))
# # get orig uuid for special metadata disaster recovery
# vm_out.write('orig_uuid=%s\n' % vm_record['uuid'])
# vm_uuid = vm_record['uuid']
# vm_out.close()
#
# Write metadata files for vdis and vbds. These end up inside of a DISK- directory.
log ('Writing disk info')
vbd_cnt = 0
for vbd in vm_record['VBDs']:
log('vbd: %s' % vbd)
vbd_record = session.xenapi.VBD.get_record(vbd)
# For each vbd, find out if its a disk
if vbd_record['type'].lower() != 'disk':
continue
vbd_record_device = vbd_record['device']
if vbd_record_device == '':
# not normal - flag as warning.
# this seems to occur on some vms that have not been started in a long while,
# after starting the vm this blank condition seems to go away.
tmp_error += 'empty vbd_record[device] on vbd: %s ' % vbd
# if device is not available then use counter as a alternate reference
vbd_cnt += 1
vbd_record_device = vbd_cnt
vdi_record = session.xenapi.VDI.get_record(vbd_record['VDI'])
log('disk: %s - begin' % vdi_record['name_label'])
# now write out the vbd info.
device_path = '%s/DISK-%s' % (tmp_full_backup_dir, vbd_record_device)
os.mkdir(device_path)
vbd_out = open('%s/vbd.cfg' % device_path, 'w')
vbd_out.write('userdevice=%s\n' % vbd_record['userdevice'])
vbd_out.write('bootable=%s\n' % vbd_record['bootable'])
vbd_out.write('mode=%s\n' % vbd_record['mode'])
vbd_out.write('type=%s\n' % vbd_record['type'])
vbd_out.write('unpluggable=%s\n' % vbd_record['unpluggable'])
vbd_out.write('empty=%s\n' % vbd_record['empty'])
# get orig uuid for special metadata disaster recovery
vbd_out.write('orig_uuid=%s\n' % vbd_record['uuid'])
# other_config and qos stuff is not backed up
vbd_out.close()
# now write out the vdi info.
vdi_out = open('%s/vdi.cfg' % device_path, 'w')
#vdi_out.write('name_label=%s\n' % vdi_record['name_label'])
vdi_out.write('name_label=%s\n' % (vdi_record['name_label']).encode("utf-8"))
# vdi_out.write('name_description=%s\n' % vdi_record['name_description'])
vdi_out.write('name_description=%s\n' % (vdi_record['name_description']).encode("utf-8"))
vdi_out.write('virtual_size=%s\n' % vdi_record['virtual_size'])
vdi_out.write('type=%s\n' % vdi_record['type'])
vdi_out.write('sharable=%s\n' % vdi_record['sharable'])
vdi_out.write('read_only=%s\n' % vdi_record['read_only'])
# get orig uuid for special metadata disaster recovery
vdi_out.write('orig_uuid=%s\n' % vdi_record['uuid'])
sr_uuid = session.xenapi.SR.get_record(vdi_record['SR'])['uuid']
vdi_out.write('orig_sr_uuid=%s\n' % sr_uuid)
# other_config and qos stuff is not backed up
vdi_out.close()
if vbd_record_device == 'xvda':
xvda_uuid = vdi_record['uuid']
xvda_name_label = vdi_record['name_label']
# Write metadata files for vifs. These are put in VIFs directory
log ('Writing VIF info')
for vif in vm_record['VIFs']:
vif_record = session.xenapi.VIF.get_record(vif)
log ('Writing VIF: %s' % vif_record['device'])
device_path = '%s/VIFs' % tmp_full_backup_dir
if (not os.path.exists(device_path)):
os.mkdir(device_path)
vif_out = open('%s/vif-%s.cfg' % (device_path, vif_record['device']), 'w')
vif_out.write('device=%s\n' % vif_record['device'])
network_name = session.xenapi.network.get_record(vif_record['network'])['name_label']
vif_out.write('network_name_label=%s\n' % network_name)
vif_out.write('MTU=%s\n' % vif_record['MTU'])
vif_out.write('MAC=%s\n' % vif_record['MAC'])
vif_out.write('other_config=%s\n' % vif_record['other_config'])
vif_out.write('orig_uuid=%s\n' % vif_record['uuid'])
vif_out.close()
return tmp_error
def final_cleanup( tmp_full_path_backup_file, tmp_backup_file_size, tmp_full_backup_dir, tmp_vm_backup_dir, tmp_vm_max_backups):
# mark this a successful backup, note: this will 'touch' a file named 'success'
# if backup size is greater than 60G, then nfs server side compression occurs
if tmp_backup_file_size > 60:
log('*** LARGE FILE > 60G: %s : %sG' % (tmp_full_path_backup_file, tmp_backup_file_size))
# forced compression via background gzip (requires nfs server side script)
open('%s/success_compress' % tmp_full_backup_dir, 'w').close()
log('*** success_compress: %s : %sG' % (tmp_full_path_backup_file, tmp_backup_file_size))
else:
open('%s/success' % tmp_full_backup_dir, 'w').close()
log('*** success: %s : %sG' % (tmp_full_path_backup_file, tmp_backup_file_size))
# Remove oldest if more than tmp_vm_max_backups
dir_to_remove = get_dir_to_remove(tmp_vm_backup_dir, tmp_vm_max_backups)
while (dir_to_remove):
log ('Deleting oldest backup %s/%s ' % (tmp_vm_backup_dir, dir_to_remove))
# remove dir - if throw exception then stop processing
shutil.rmtree(tmp_vm_backup_dir + '/' + dir_to_remove)
dir_to_remove = get_dir_to_remove(tmp_vm_backup_dir, tmp_vm_max_backups)
#### need to just feed in directory and find oldest named subdirectory
### def pre_cleanup( tmp_full_path_backup_file, tmp_full_backup_dir, tmp_vm_backup_dir, tmp_vm_max_backups):
def pre_cleanup(tmp_vm_backup_dir, tmp_vm_max_backups):
#print ' ==== tmp_full_backup_dir: %s' % tmp_full_backup_dir
#print ' ==== tmp_vm_backup_dir: %s' % tmp_vm_backup_dir
#print ' ==== tmp_vm_max_backups: %d' % tmp_vm_max_backups
log('success identifying directory : %s ' % tmp_vm_backup_dir)
# Remove oldest if more than tmp_vm_max_backups -1
pre_vm_max_backups = tmp_vm_max_backups - 1
log ("pre_VM_max_backups: %s " % pre_vm_max_backups)
if pre_vm_max_backups < 1:
log ('No pre_cleanup needed for %s ' % tmp_vm_backup_dir)
else:
dir_to_remove = get_dir_to_remove(tmp_vm_backup_dir, tmp_vm_max_backups)
while (dir_to_remove):
log ('Deleting oldest backup %s/%s ' % (tmp_vm_backup_dir, dir_to_remove))
# remove dir - if throw exception then stop processing
shutil.rmtree(tmp_vm_backup_dir + '/' + dir_to_remove)
dir_to_remove = get_dir_to_remove(tmp_vm_backup_dir, tmp_vm_max_backups)
# cleanup old unsuccessful backup and create new full_backup_dir
def process_backup_dir(tmp_vm_backup_dir):
if (not os.path.exists(tmp_vm_backup_dir)):
# Create new dir - if throw exception then stop processing
os.mkdir(tmp_vm_backup_dir)
# if last backup was not successful, then delete it
log ('Check for last **unsuccessful** backup: %s' % tmp_vm_backup_dir)
dir_not_success = get_last_backup_dir_that_failed(tmp_vm_backup_dir)
if (dir_not_success):
#if (not os.path.exists(tmp_vm_backup_dir + '/' + dir_not_success + '/fail')):
log ('Delete last **unsuccessful** backup %s/%s ' % (tmp_vm_backup_dir, dir_not_success))
# remove last unseccessful backup - if throw exception then stop processing
shutil.rmtree(tmp_vm_backup_dir + '/' + dir_not_success)
# create new backup dir
return create_full_backup_dir(tmp_vm_backup_dir)
# Setup full backup dir structure
def create_full_backup_dir(vm_base_path):
# Check that directory exists
if not os.path.exists(vm_base_path):
# Create new dir - if throw exception then stop processing
os.mkdir(vm_base_path)
date = datetime.datetime.today()
tmp_backup_dir = BACKUP_DIR_PATTERN \
% (vm_base_path, date.year, date.month, date.day, date.hour, date.minute, date.second)
log('new backup_dir: %s' % tmp_backup_dir)
if not os.path.exists(tmp_backup_dir):
# Create new dir - if throw exception then stop processing
os.mkdir(tmp_backup_dir)
return tmp_backup_dir
# Setup meta dir structure
def get_meta_path(base_path):
# Check that directory exists
if not os.path.exists(base_path):
# Create new dir
try:
os.mkdir(base_path)
except OSError, error:
log('ERROR creating directory %s : %s' % (base_path, error.as_string()))
return False
date = datetime.datetime.today()
backup_path = '%s/pool_db_%04d%02d%02d-%02d%02d%02d.dump' \
% (base_path, date.year, date.month, date.day, date.hour, date.minute, date.second)
return backup_path
def get_dir_to_remove(path, numbackups):
# Find oldest backup and select for deletion
dirs = os.listdir(path)
dirs.sort()
if (len(dirs) > numbackups and len(dirs) > 1):
return dirs[0]
else:
return False
def get_last_backup_dir_that_failed(path):
# if the last backup dir was not success, then return that backup dir
dirs = os.listdir(path)
if (len(dirs) <= 1):
return False
dirs.sort()
# note: dirs[-1] is the last entry
#print "==== dirs that failed: %s" % dirs
if (not os.path.exists(path + '/' + dirs[-1] + '/success')) and \
(not os.path.exists(path + '/' + dirs[-1] + '/success_restore')) and \
(not os.path.exists(path + '/' + dirs[-1] + '/success_compress' )) and \
(not os.path.exists(path + '/' + dirs[-1] + '/success_compressing' )):
return dirs[-1]
else:
return False
def check_all_backups_success(path):
# expect at least one backup dir, and all should be successful
dirs = os.listdir(path)
if (len(dirs) == 0):
return False
for dir in dirs:
if (not os.path.exists(path + '/' + dir + '/success' )) and \
(not os.path.exists(path + '/' + dir + '/success_restore' )) and \
(not os.path.exists(path + '/' + dir + '/success_compress' )) and \
(not os.path.exists(path + '/' + dir + '/success_compressing' )):
log("WARNING: directory not successful - %s" % dir)
return False
return True
def backup_pool_metadata(svr_name):
# xe-backup-metadata can only run on master
if not is_xe_master():
log('** ignore: NOT master')
return True
metadata_base = os.path.join(config['backup_dir'], 'METADATA_' + svr_name)
metadata_file = get_meta_path(metadata_base)
cmd = "%s/xe pool-dump-database file-name='%s'" % (xe_path, metadata_file)
log(cmd)
if run_log_out_wait_rc(cmd) != 0:
log('ERROR failed to backup pool metadata')
return False
return True
# some run notes with xe return code and output examples
# xe vm-lisX -> error .returncode=1 w/ error msg
# xe vm-list name-label=BAD-vm-name -> success .returncode=0 with no output
# xe pool-dump-database file-name=<dup-file-already-exists>
# -> error .returncode=1 w/ error msg
def run_log_out_wait_rc(cmd, log_w_timestamp=True):
child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
line = child.stdout.readline()
while line:
log(line.rstrip("\n"), log_w_timestamp)
line = child.stdout.readline()
return child.wait()
def run_get_lastline(cmd):
# exec cmd - expect 1 line output from cmd
# return last line
f = os.popen(cmd)
resp = ''
for line in f.readlines():
resp = line.rstrip("\n")
return resp
def get_os_version(uuid):
cmd = "%s/xe vm-list uuid='%s' params=os-version | /bin/grep 'os-version' | /bin/awk -F'name: ' '{print $2}' | /bin/awk -F'|' '{print $1}' | /bin/awk -F';' '{print $1}'" % (xe_path, uuid)
return run_get_lastline(cmd)
def df_snapshots(log_msg):
log(log_msg)
f = os.popen('df -Th %s' % config['backup_dir'])
for line in f.readlines():
line = line.rstrip("\n")
log(line)
def send_email(to, subject, body_fname):
smtp_send_retries = 3
smtp_send_attempt = 0
message = open('%s' % body_fname, 'r').read()
msg = MIMEText(message)
msg['subject'] = subject
msg['From'] = MAIL_FROM_ADDR
msg['To'] = to
while smtp_send_attempt < smtp_send_retries:
smtp_send_attempt += 1
if smtp_send_attempt > smtp_send_retries:
print("Send email count limit exceeded")
sys.exit(1)
try:
# note if using an ipaddress in MAIL_SMTP_SERVER,
# then may require smtplib.SMTP(MAIL_SMTP_SERVER, local_hostname="localhost")
## Optional use of SMTP user authentication via TLS
##
## If so, comment out the next line of code and uncomment/configure
## the next block of code. Note that different SMTP servers will require
## different username options, such as the plain username, the
## domain\username, etc. The "From" email address entry must be a valid
## email address that can be authenticated and should be configured
## in the MAIL_FROM_ADDR variable along with MAIL_SMTP_SERVER early in
## the script. Note that some SMTP servers might use port 465 instead of 587.
s = smtplib.SMTP(MAIL_SMTP_SERVER)
#### start block
#username = 'MyLogin'
#password = 'MyPassword'
#s = smtplib.SMTP(MAIL_SMTP_SERVER, 587)
#s.ehlo()
#s.starttls()
#s.login(username, password)
#### end block
s.sendmail(MAIL_FROM_ADDR, to.split(','), msg.as_string())
s.quit()
break
except socket.error, e:
print("Exception: socket.error - %s" %e)
time.sleep(5)
except smtplib.SMTPException, e:
print("Exception: SMTPException - %s" %e.message)
time.sleep(5)
def is_xe_master():
# test to see if we are running on xe master
cmd = '%s/xe pool-list params=master --minimal' % xe_path
master_uuid = run_get_lastline(cmd)
hostname = os.uname()[1]
cmd = '%s/xe host-list name-label=%s --minimal' % (xe_path, hostname)
host_uuid = run_get_lastline(cmd)
if host_uuid == master_uuid:
return True
return False
def is_config_valid():
if not isInt(config['pool_db_backup']):
print 'ERROR: config pool_db_backup non-numeric -> %s' % config['pool_db_backup']
return False
if int(config['pool_db_backup']) != 0 and int(config['pool_db_backup']) != 1:
print 'ERROR: config pool_db_backup out of range -> %s' % config['pool_db_backup']
return False
if not isInt(config['max_backups']):
print 'ERROR: config max_backups non-numeric -> %s' % config['max_backups']
return False
if int(config['max_backups']) < 1:
print 'ERROR: config max_backups out of range -> %s' % config['max_backups']
return False
if config['vdi_export_format'] != 'raw' and config['vdi_export_format'] != 'vhd':
print 'ERROR: config vdi_export_format invalid -> %s' % config['vdi_export_format']
return False
if not os.path.exists(config['backup_dir']):
print 'ERROR: config backup_dir does not exist -> %s' % config['backup_dir']
return False
tmp_return = True
for vm_parm in config['vdi-export']:
if not is_vm_backups_valid(vm_parm):
print 'ERROR: vm_max_backup is invalid - %s' % vm_parm
tmp_return = False
for vm_parm in config['vm-export']:
if not is_vm_backups_valid(vm_parm):
print 'ERROR: vm_max_backup is invalid - %s' % vm_parm
tmp_return = False
return tmp_return
def config_load(path):
return_value = True
config_file = open(path, 'r')
for line in config_file:
if (not line.startswith('#') and len(line.strip()) > 0):
(key,value) = line.strip().split('=')
key = key.strip()
value = value.strip()