1 | #!/usr/bin/python |
---|
2 | # Migrates the machine passed as arguments from the dev cluster. |
---|
3 | # To be run on the prod cluster. |
---|
4 | |
---|
5 | from invirt import remctl as r |
---|
6 | from lib import database |
---|
7 | import subprocess |
---|
8 | import sys |
---|
9 | import time |
---|
10 | import os |
---|
11 | |
---|
12 | sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) |
---|
13 | |
---|
14 | kinit = dict(principal='host/aperture-science.mit.edu', keytab='/etc/krb5.keytab') |
---|
15 | |
---|
16 | dev_db_uri = 'postgres://sipb-xen@sipb-xen-dev.mit.edu/sipb_xen' |
---|
17 | database.connect(dev_db_uri) |
---|
18 | dev_sess = database.session |
---|
19 | |
---|
20 | database.connect() |
---|
21 | prod_sess = database.session |
---|
22 | |
---|
23 | ## dump from dev db |
---|
24 | def take_data(machine_name): |
---|
25 | dev_sess.begin() |
---|
26 | machine = dev_sess.query(database.Machine).filter_by(name=machine_name).one() |
---|
27 | |
---|
28 | # Clean out the ACL just so we don't have to think about it |
---|
29 | machine.acl = [] |
---|
30 | dev_sess.update(machine) |
---|
31 | |
---|
32 | print 'VM Info:' |
---|
33 | print ' name: %s' % machine.name |
---|
34 | print ' description: %s' % machine.description |
---|
35 | print ' cpus: %s' % machine.cpus |
---|
36 | print ' memory: %s' % machine.memory |
---|
37 | print ' owner: %s' % machine.owner |
---|
38 | print ' contact: %s' % machine.contact |
---|
39 | print ' administrator: %s' % machine.administrator |
---|
40 | print ' uuid: %s' % machine.uuid |
---|
41 | print ' type: %s' % machine.type.type_id |
---|
42 | print ' autorestart: %s' % machine.autorestart |
---|
43 | print ' adminable: %s' % machine.adminable |
---|
44 | print ' Disks:' |
---|
45 | for disk in machine.disks: |
---|
46 | print ' - %s (%s)' % (disk.guest_device_name, disk.size) |
---|
47 | print ' NICs:' |
---|
48 | for nic in machine.nics: |
---|
49 | print ' - %s, %s, %s' % (nic.mac_addr, nic.ip, nic.hostname) |
---|
50 | print '===============================================' |
---|
51 | print |
---|
52 | |
---|
53 | disks = machine.disks |
---|
54 | nics = machine.nics |
---|
55 | for r in disks + nics + [machine]: |
---|
56 | dev_sess.delete(r) |
---|
57 | |
---|
58 | dev_sess.flush() |
---|
59 | |
---|
60 | for r in disks + nics + [machine]: |
---|
61 | dev_sess.expunge(r) |
---|
62 | del r._instance_key |
---|
63 | |
---|
64 | return machine |
---|
65 | |
---|
66 | ## add to prod db |
---|
67 | def restore_data(machine, session): |
---|
68 | # The machine's type is still the one attached to the dev database; |
---|
69 | # get the right one |
---|
70 | machine.type = session.query(database.Type).filter_by(type_id=machine.type.type_id).one() |
---|
71 | session.begin() |
---|
72 | session.save(machine) |
---|
73 | session.commit() |
---|
74 | |
---|
75 | def migrate_vm(machine_name): |
---|
76 | # Power off the VM on dev |
---|
77 | # |
---|
78 | # This has to be done first, because once the machine is deleted |
---|
79 | # from the database, we can't remctl for it anymore |
---|
80 | out, err = r.remctl('xvm-remote.mit.edu', 'control', machine_name, 'destroy', err=True, **kinit) |
---|
81 | print out |
---|
82 | |
---|
83 | machine = take_data(machine_name) |
---|
84 | subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m', |
---|
85 | 'Migrating %s, disk size %0.2fG...' % (machine.name, sum(disk.size for disk in machine.disks) / 1024.0)]) |
---|
86 | |
---|
87 | success = True |
---|
88 | ## copy disk image... copy, copy... |
---|
89 | for disk in machine.disks: |
---|
90 | lvname='d_%s_%s' % (machine.name, disk.guest_device_name) |
---|
91 | |
---|
92 | if 0 != subprocess.call(['lvcreate', '-L%sM' % str(disk.size), '-n', lvname, 'xenvg']): |
---|
93 | success = False |
---|
94 | break |
---|
95 | |
---|
96 | ssh = subprocess.Popen(['rsh', |
---|
97 | '10.6.0.165', |
---|
98 | 'dd', 'if=/dev/xenvg/%s' % lvname, 'bs=1M'], |
---|
99 | stdout=subprocess.PIPE) |
---|
100 | dd = subprocess.Popen(['dd', 'of=/dev/xenvg/%s' % lvname, 'bs=1M'], |
---|
101 | stdin=ssh.stdout) |
---|
102 | if 0 != dd.wait(): |
---|
103 | success = False |
---|
104 | break |
---|
105 | if 0 != ssh.wait(): |
---|
106 | success = False |
---|
107 | break |
---|
108 | |
---|
109 | if not success: |
---|
110 | dev_sess.rollback() |
---|
111 | |
---|
112 | print '===============================================' |
---|
113 | print 'ERROR: VM %s failed to migrate' % machine.name |
---|
114 | print '===============================================' |
---|
115 | else: |
---|
116 | restore_data(machine, prod_sess) |
---|
117 | dev_sess.commit() |
---|
118 | |
---|
119 | return success |
---|
120 | |
---|
121 | if __name__ == '__main__': |
---|
122 | while True: |
---|
123 | r.checkKinit(**kinit) |
---|
124 | p = subprocess.Popen(['curl', '-s', '-k', '--negotiate', '-u', ':', 'https://xvm.mit.edu:442/offlist'], stdout=subprocess.PIPE) |
---|
125 | if 0 != p.wait(): |
---|
126 | subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m', |
---|
127 | 'Failed to get list of remaining VMs. Will try again in 15 seconds']) |
---|
128 | time.sleep(15) |
---|
129 | continue |
---|
130 | |
---|
131 | next_line = p.stdout.read().split('\n')[0] |
---|
132 | if next_line == '': |
---|
133 | subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m', |
---|
134 | 'XVM migration complete']) |
---|
135 | break |
---|
136 | |
---|
137 | next, uptime = next_line.split('\t')[:2] |
---|
138 | |
---|
139 | print '===============================================' |
---|
140 | print 'Migrating %s' % next |
---|
141 | print '===============================================' |
---|
142 | if not migrate_vm(next): |
---|
143 | subprocess.call(['zwrite', '-d', '-c', 'xvm', '-i', 'migration', '-s', 'XVM Migration Script', '-m', |
---|
144 | 'Error in migrating %s' % next]) |
---|
145 | sys.exit(1) |
---|
146 | if uptime.strip() != '': |
---|
147 | r.remctl('xvm-remote-dev.mit.edu', 'control', next, 'create', **kinit) |
---|
148 | |
---|
149 | subprocess.call(['zwrite', '-d', '-c', 'xvm-auto', '-i', 'migration', '-s', 'XVM Migration Script', '-m', |
---|
150 | 'done']) |
---|