OpenStack资源回收

实现原理

更新实例元数据后OpenStack会往nova数据库的instance_metadata表的key、value列中插入相关的信息。这样我们可以自定义一个expired_time的元数据参数,用于表示该实例的过期时间。然后通过定期从数据库获取每个实例的该参数值,并与当前时间进行比较,若到期则通过调用openstack client的命令关闭实例,若到期未修改该参数值且超过7天则自动删除实例。同时在实例即将到期前2天到删除实例这段时间向用户发送邮件提醒。



1
2
3
4
5
6
MariaDB [nova]> select instance_uuid,`key`,value from instance_metadata where `key`="expired_time";
+--------------------------------------+--------------+----------+
| instance_uuid | key | value |
+--------------------------------------+--------------+----------+
| 00a52695-c3df-4633-9797-db268e37187d | expired_time | 20181231 |
+--------------------------------------+--------------+----------+

若实例需要永久使用,则需要向运维发起申请,并由运维人员添加白名单whitelist(由于实例名非唯一标识可重复,故需要指定实例ID)

1
2
3
4
5
c639961f-e8b2-4655-8be0-f7e2b808f06b    opsbase-test-zhouyp
07527701-4a45-4349-93cb-ba1924af7c81 ydbc-test-pgxa-1
d4b21c62-2baf-4963-bb42-74cda3f67295 ydbc-test-pgxa-2
7e5dc6e1-54bf-4147-9c98-d6c64a8aa02b ydbc-test-pgxa-3
76874a2b-0f1d-464a-88e6-8e56bde9724d ydbc-test-pgxa-4

实现脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/bin/bash
whitelist=$(grep -v ^"#" whitelist |awk '{print $1}')
expired_time_default=`date -d "+ 1 month" +%Y%m%d`
select_instance_all="select created_at,uuid,user_id,display_name,vm_state from instances where vm_state !='deleted' order by created_at DESC"
select_instance_with_expired="select a.uuid,a.user_id,a.display_name,b.value,a.vm_state from instances a,instance_metadata b where a.vm_state !='deleted' and b.deleted=0 and a.uuid=b.instance_uuid and b.key='expired_time' order by b.value,a.user_id"
reference_url="http://docs.domain.com/运维文档/document/OpenStack/01.虚拟机资源管理.html#openstack虚拟机续期步骤"
. /root/admin-openrc.sh

function fetch_instance_all(){
mysql -h10.200.111.250 -uroot -pav9ECgn40cE8C7AXNtauXuPpKysC2C5INGxwmj1U nova -t -N -e "$select_instance_all" > list_all
sed -i -e '1d' -e '$d' list_all
}
function fetch_instance_with_expired(){
mysql -h10.200.111.250 -uroot -pav9ECgn40cE8C7AXNtauXuPpKysC2C5INGxwmj1U nova -t -N -e "$select_instance_with_expired" > list
sed -i -e '1d' -e '$d' list
}
function add_expired_time(){
for id in $(awk -F'|' '{print $3}' list_all)
do
create_time=`grep $id list_all | awk -F"|" '{print $2}'`
create_interval=`echo "$(date +%s -d "-8 hour") $(date -d "$create_time" +%s) 3600" | awk '{printf("%0.1f\n",($1-$2)/3600)}'`

if [ `echo "$create_interval < 24" | bc` -eq 1 ];then
echo "openstack server set --property expired_time=$expired_time_default $id" >> resource_recycle.log
openstack server set --property expired_time=$expired_time_default $id
fi
done
}

function resource_recycle(){
for id in $(awk -F'|' '{print $2}' list)
do
echo $whitelist | grep $id > /dev/null
if [ "$?" -eq 0 ];then
continue
fi

vm_state=`grep $id list |awk -F'|' '{print $6}'`
expired_time=`grep $id list |awk -F'|' '{print $5}'`
instance_name=`grep $id list |awk -F'|' '{print $4}'`
mail=`grep $id list |awk -F'|' '{print $3}'`

expired_time_19700101=$(date -d "$expired_time" +%s)
# check expired_time is valid
if [ -z "$expired_time_19700101" ];then
echo "instance:$instance_name expired_time is invalid,set to $expired_time_default force" >> resource_recycle.log
echo "openstack server set --property expired_time=$expired_time_default $id" >> resource_recycle.log
openstack server set --property expired_time=$expired_time_default $id
expired_time_19700101=$(date -d "$expired_time_default" +%s)
echo "mail to $mail for 虚拟机到期时间格式无效提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]设置的expired_time:$expired_time 为无效日期,强制设为$expired_time_default" | mailx -s "虚拟机到期时间格式无效提醒" $mail
fi

current_19700101=$(date +%s)
remain_day=`echo "$expired_time_19700101 $current_19700101 86400" | awk '{printf("%0.1f\n",($1-$2)/$3)}'`

# remain_day > 31
if [ `echo "$remain_day > 31" | bc` -eq 1 ];then
remain_day=`echo "$(date +%s -d '+1 month') $(date +%s) 86400" | awk '{printf("%0.1f\n",($1-$2)/$3)}'`
echo "instance:$instance_name remain_day exceed 1 month,set to $expired_time_default force" >> resource_recycle.log
echo "openstack server set --property expired_time=$expired_time_default $id" >> resource_recycle.log
openstack server set --property expired_time=$expired_time_default $id
echo "mail to $mail for 虚拟机到期时间无效提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]设置的expired_time:$expired_time 超过31天,且未申请加入白名单,强制设为$expired_time_default" | mailx -s "虚拟机到期时间无效提醒" $mail

# remain_day = 7 or 0 < remain_day ≤ 2,send mail
elif [ `echo "$remain_day == 7" | bc` -eq 1 -o `echo "$remain_day <= 2" | bc` -eq 1 -a `echo "$remain_day > 0" | bc` -eq 1 ];then
echo "instance:$instance_name have "$remain_day" day remain,and will shutdown after $remain_day day" >> resource_recycle.log
echo "mail to $mail for 虚拟机即将到期提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]剩余 $remain_day 天到期,到期后机器将停止服务,并在2周后自动删除,如需继续使用请及时续期。\n具体操作请参考:$reference_url"| mailx -s "虚拟机即将到期提醒" $mail

# -14 < remain_day ≤ 0,shutdown if instance is active and send mail
elif [ `echo "$remain_day <= 0" | bc` -eq 1 -a `echo "$remain_day > -14" | bc` -eq 1 ];then
delete_time=`echo "$remain_day 14" | awk '{printf("%0.1f\n",$1+$2)}'`
remain_day_abs=`echo "$remain_day" | awk '{printf("%0.1f\n",0-$1)}'`
if [ $vm_state != "stopped" ];then
echo "instance:$instance_name have $remain_day day remain,begin to shutdown" >> resource_recycle.log
echo "openstack server stop $id" >> resource_recycle.log
openstack server stop $id
sleep 30
vm_state=`openstack server show $id|grep vm_state|awk '{print $4}'`
echo "mail to $mail for 虚拟机到期关闭即将删除提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]已过期 $remain_day_abs 天并停止服务,将在$delete_time天后自动删除,如需继续使用请及时续期。\n具体操作请参考:$reference_url"| mailx -s "虚拟机到期关闭即将删除提醒" $mail
fi

if [ $vm_state = "stopped" ];then
echo "instance:$instance_name is already been shutdown,and will delete after $delete_time day" >> resource_recycle.log

else
echo "openstack stop instance:$instance_name failed" >> resource_recycle.log
echo "OpenStack实例[ $instance_name ]到期自动关闭失败"| mailx -s "虚拟机到期停机失败" zhouyp@domain.com
fi

# remain_day ≤ -14,delete if instance is stopped and send mail
elif [ `echo "$remain_day <= -14" | bc` -eq 1 ];then
if [ $vm_state = "active" ];then
echo instance:$instance_name expired more than 2 weeks but is active >> resource_recycle.log
echo "mail to $mail for 虚拟机过期超过2周本应删除但仍开机未删除提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]已过期超过2周本应立即自动删除,但由于仍处于开机状态,已关闭虚拟机并于第二天凌晨删除,请确认该虚拟机是否需要续期。\n具体续期操作请参考:$reference_url"| mailx -s "虚拟机过期超过2周本应删除但仍开机未删除提醒" $mail

echo "openstack server stop $id" >> resource_recycle.log
openstack server stop $id
echo "OpenStack实例[ $instance_name ]已过期超过2周但仍处于开机状态"| mailx -s "虚拟机过期超过2周本应删除但仍开机未删除提醒" zhouyp@domain.com
else
echo "openstack server delete $id" >> resource_recycle.log
openstack server delete $id
sleep 30
echo "instance:$instance_name is already been delete" >> resource_recycle.log
echo "mail to $mail for 虚拟机删除提醒" >> resource_recycle.log
sleep 2
echo -e "您好:\n您的OpenStack实例[ $instance_name ]已过期超过2周并自动删除"| mailx -s "虚拟机删除提醒" $mail
vm_state=`openstack server show $id|grep vm_state|awk '{print $4}'`
fi
fi
echo "| $mail | \`$instance_name\` | $vm_state | $remain_day | ">> report.md
done
}

function whitelist(){
for id in $(grep -v ^"#" whitelist|awk '{print $1}')
do
vm_state=`grep $id list |awk -F'|' '{print $6}'`
remain_day=forever
instance_name=`grep $id list |awk -F'|' '{print $4}'`
mail=`grep $id list |awk -F'|' '{print $3}'`

echo "| $mail | \`$instance_name\` | $vm_state | **\`"$remain_day"\`** |" >> report.md
done
}

function main () {
fetch_instance_all
add_expired_time
fetch_instance_with_expired
resource_recycle
whitelist
}

cat > report.md <<EOF
| 负责人 | 实例 |状态|剩余天数 |
| --- | --- | ---| -- |
EOF

echo "============================ Begin: `date +%Y.%m.%d-%H:%M:%S` ============================" >> resource_recycle.log
echo "============================ Begin: `date +%Y.%m.%d-%H:%M:%S` ============================" >> mail.log
echo "============================ Begin: `date +%Y.%m.%d-%H:%M:%S` ============================" >> dead.letter
main $@
echo "============================ End: `date +%Y.%m.%d-%H:%M:%S` ============================" >> resource_recycle.log
echo "============================ End: `date +%Y.%m.%d-%H:%M:%S` ============================" >> mail.log
echo "============================ End: `date +%Y.%m.%d-%H:%M:%S` ============================" >> dead.letter

实现效果

邮件提醒

1
2
3
4
5
6
7
8
9
10
11
12
# yum -y install mailx
# vi /etc/mail.rc
set encoding=8bit
set DEAD=~/resource_recycle/dead.letter
set record=~/resource_recycle/mail.log
set from="xxx@domain.com"
set smtp=mail.domain.com:587
set smtp-auth-user="xxxx@domain.com"
set smtp-auth-password="xxxxxxxx"
set smtp-use-starttls
set ssl-verify=ignore
set nss-config-dir=/root/.certs

生成报表

脚本执行后将生产一份repo.md的文件,可查看所有实例的到期时间,若实例到期需要续期,则可更新指定实例的元数据expired_time参数

负责人 实例 状态 剩余天数
cxh@domain.com uc-protobuf-gw-cxh active 29.0
laiyl@domain.com laiyl_wangb_test active 29.0
laiyl@domain.com laiyl_linzw_联盟2 active 29.0
laiyl@domain.com yms-fengsh active 29.0
zhouyp@domain.com ydbc-test-b-yoperlog active 29.0
zhouyp@domain.com ydbc-test-elasticsearch-1 active 29.0
zhouyp@domain.com opsbase-test-zhouyp active forever
zhouyp@domain.com ydbc-test-pgxa-1 active forever
zhouyp@domain.com ydbc-test-pgxa-2 active forever
zhouyp@domain.com ydbc-test-pgxa-3 active forever
zhouyp@domain.com ydbc-test-pgxa-4 active forever
坚持原创技术分享,您的支持将鼓励我继续创作!
0%