You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
328 lines
9.0 KiB
328 lines
9.0 KiB
#!/usr/bin/perl -w
|
|
|
|
use strict;
|
|
use warnings;
|
|
use Getopt::Long;
|
|
use Config::Simple;
|
|
use LWP::UserAgent;
|
|
use JSON;
|
|
use PVE::QemuServer;
|
|
use PVE::LXC::Config;
|
|
use Array::Diff;
|
|
use Net::Address::IP::Local;
|
|
use Net::Route::Table;
|
|
use NetAddr::IP;
|
|
use Data::Validate::IP qw(is_ipv4);
|
|
use Data::Dumper;
|
|
|
|
$| = 1;
|
|
|
|
my $config = '/etc/pve-online.conf';
|
|
my ($update_routes, $update_gre, $migrate_ipfo) = undef;
|
|
|
|
GetOptions(
|
|
"config=s" => \$config,
|
|
"update-routes" => \$update_routes,
|
|
"update-gre" => \$update_gre,
|
|
"migrate-ipfo=i" => \$migrate_ipfo
|
|
);
|
|
|
|
# Config can be stored in /etc/pve
|
|
# Lets wait a bit for it to be available
|
|
if ($config =~ m|^/etc/pve|){
|
|
my $t = 0;
|
|
while ($t < 120){
|
|
if (!-e $config){
|
|
print "$config not yet available\n";
|
|
sleep 2;
|
|
$t += 2;
|
|
} else {
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
if (!-e $config){
|
|
die "$config doesn't exist\n";
|
|
}
|
|
|
|
my $cfg = new Config::Simple;
|
|
$cfg->read($config);
|
|
my $conf = $cfg->get_block('general');
|
|
my $lwp = new LWP::UserAgent;
|
|
my $online_id = undef;
|
|
|
|
if (!$conf){
|
|
die "No general section found in $config\n";
|
|
}
|
|
if (!$conf->{online_api}){
|
|
die "No online_api defined in $config\n";
|
|
}
|
|
|
|
# Set some defaults
|
|
$conf->{wan_bridge} ||= 'vmbr1';
|
|
$conf->{migrate_flush_arp} ||= 'yes';
|
|
|
|
if ($update_routes){
|
|
update_routes();
|
|
}
|
|
if ($update_gre){
|
|
if (defined $conf->{vlan_bridge}){
|
|
update_gre();
|
|
} else {
|
|
print "No VLAN bridge defined. Don't setup GRE tunnels\n";
|
|
}
|
|
}
|
|
if ($migrate_ipfo){
|
|
my $ipfo = [];
|
|
# We parse the description field which contains the list of ipfo
|
|
# attached to this VM
|
|
foreach my $line ( split("\n", get_guest_conf($migrate_ipfo)->{description} || "") ){
|
|
if ($line =~ m/^\s*ipfo\d*:\s+(\d+\.\d+\.\d+\.\d+)/){
|
|
my $candidate = $1;
|
|
if (is_ipv4($candidate)){
|
|
push @{$ipfo}, $candidate;
|
|
print "Found IP $candidate assigned to guest $migrate_ipfo\n";
|
|
} else {
|
|
print "Found $candidate assigned to guest $migrate_ipfo which doesn't look like a valid IP\n";
|
|
}
|
|
}
|
|
}
|
|
$online_id ||= get_online_id();
|
|
# Now we check if we need to migrate IPFO on the local server
|
|
my $ipfo_diff = Array::Diff->diff(get_ip_fo($online_id), $ipfo);
|
|
if (scalar @{$ipfo_diff->added} > 0){
|
|
print "Update needed. " . join(' ', @{$ipfo_diff->added}) . " should be redirected on server $online_id\n";
|
|
redirect_ipfo($ipfo_diff->added);
|
|
update_routes();
|
|
if ( $ENV{PVE_GUEST_TYPE} ne 'lxc' and $conf->{migrate_flush_arp} =~ m/^1|yes|true|on$/i ){
|
|
set_guest_nic_down_up($migrate_ipfo);
|
|
}
|
|
}
|
|
}
|
|
|
|
#####################################
|
|
# Sub routines
|
|
#####################################
|
|
|
|
# Query Online's API
|
|
sub query_online_api{
|
|
my $uri = shift;
|
|
$uri = '/api/v1' . $uri if ($uri !~ m|^/api/v1|);
|
|
my $response = $lwp->get("https://api.online.net$uri",
|
|
"Authorization" => "Authorization: Bearer $conf->{online_api}",
|
|
);
|
|
unless ($response->is_success){
|
|
die "an error occured while querying the API" .
|
|
"The error is: " . $response->status_line;
|
|
}
|
|
return from_json($response->content);
|
|
}
|
|
|
|
# Update routes for ARP Proxy
|
|
sub update_routes {
|
|
$online_id ||= get_online_id();
|
|
# This is the list of IP which we should have
|
|
# on the routing table
|
|
my $routes_online = get_ip_fo($online_id);
|
|
|
|
# This is the actual list of IP for which we have routes
|
|
my $routes_local = get_local_routes();
|
|
|
|
# Now, we have to remove routes for those in $routes_local but not in $routes_online
|
|
# And add routes for those in $routes_online but not in $routes_local
|
|
my $diff = Array::Diff->diff($routes_online, $routes_local);
|
|
foreach my $route (@{$diff->added}){
|
|
next if (grep { $_ eq $route } @{$diff->deleted});
|
|
print "Removing route for $route\n";
|
|
system(
|
|
'/sbin/ip',
|
|
'route',
|
|
'del',
|
|
$route,
|
|
'dev',
|
|
$conf->{wan_bridge}
|
|
);
|
|
}
|
|
foreach my $route (@{$diff->deleted}){
|
|
next if (grep { $_ eq $route } @{$diff->added});
|
|
print "Adding route for $route\n";
|
|
system(
|
|
'/sbin/ip',
|
|
'route',
|
|
'add',
|
|
$route . '/32',
|
|
'dev',
|
|
$conf->{wan_bridge}
|
|
);
|
|
}
|
|
}
|
|
|
|
# Get the list of routes defined on $conf->{wan_bridge}
|
|
sub get_local_routes {
|
|
my $ip = [];
|
|
my $routes = Net::Route::Table->from_system();
|
|
foreach my $route (@{$routes->all_routes()}){
|
|
if ($route->{interface} eq $conf->{wan_bridge} and $route->destination()->masklen() == 32){
|
|
push @{$ip}, $route->destination()->addr();
|
|
}
|
|
}
|
|
return $ip;
|
|
}
|
|
|
|
# Get the list of IP failover assigned to a server. Taks a server ID as only arg
|
|
sub get_ip_fo {
|
|
my $srv_id = shift;
|
|
return get_srv_info($srv_id)->{network}->{ipfo};
|
|
}
|
|
|
|
# Return server info
|
|
sub get_srv_info {
|
|
my $srv_id = shift;
|
|
return query_online_api('/server/info/' . $srv_id);
|
|
}
|
|
|
|
# Return this server's public IP
|
|
sub get_public_ip {
|
|
return Net::Address::IP::Local->public_ipv4;
|
|
}
|
|
|
|
# Return Online's server id
|
|
sub get_online_id {
|
|
if (-e '/tmp/online_id'){
|
|
open my $id_file, '</tmp/online_id';
|
|
$online_id = <$id_file>;
|
|
close $id_file;
|
|
} else {
|
|
my $ip = get_public_ip();
|
|
foreach my $srv (@{query_online_api('/server')}){
|
|
my $info = query_online_api($srv);
|
|
if ($info->{network}->{ip}[0] eq $ip){
|
|
$online_id = $info->{id};
|
|
last;
|
|
}
|
|
}
|
|
open my $id_file, ">/tmp/online_id";
|
|
print $id_file $online_id;
|
|
close $id_file;
|
|
}
|
|
print "My Online's ID is $online_id\n";
|
|
return $online_id;
|
|
}
|
|
|
|
sub update_gre {
|
|
# We have to setup GRE tunnels with all other members
|
|
# to connect $conf->{vlan_bridge} between every nodes
|
|
# Something like
|
|
# ovs-vsctl add-port vmbr1 gre0 -- set interface gre0 type=gre options:remote_ip=''10.29.254.2''
|
|
# We just have to automate this for every nodes of the cluster
|
|
print "Getting cluster status...\n";
|
|
my $members = get_cluster_members();
|
|
print "Found " . scalar @{$members} . " members\n";
|
|
my $gre = 0;
|
|
|
|
print "Counting GRE ports...\n";
|
|
my @ports = qx(ovs-vsctl list-ports $conf->{vlan_bridge});
|
|
my @gre_ports = grep { $_ =~ m/^gre/ } @ports;
|
|
print "Found " . scalar @gre_ports . " GRE ports\n";
|
|
|
|
if (scalar @gre_ports ne scalar @{$members} - 1){
|
|
print "We need to update GRE tunnels\n";
|
|
# Remove all greX ports from the VLAN bridge
|
|
foreach my $port ( @ports ){
|
|
chomp($port);
|
|
next unless ($port =~ m/^gre\d+$/);
|
|
print "Removing port $port from $conf->{vlan_bridge}\n";
|
|
system(
|
|
'ovs-vsctl',
|
|
'del-port',
|
|
$conf->{vlan_bridge},
|
|
$port
|
|
);
|
|
}
|
|
# And setup one GRE tunnel per node
|
|
foreach my $member (@{$members}){
|
|
# We must skip our own node
|
|
if (Net::Address::IP::Local->connected_to($member) ne $member){
|
|
print "Adding GRE interface gre$gre to tunnel with $member\n";
|
|
system(
|
|
'ovs-vsctl',
|
|
'add-port',
|
|
$conf->{vlan_bridge},
|
|
'gre' . $gre,
|
|
'--',
|
|
'set',
|
|
'interface',
|
|
'gre' . $gre,
|
|
'type=gre',
|
|
'options:remote_ip=' . $member
|
|
);
|
|
$gre++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Get the list of members of this proxmox cluster
|
|
sub get_cluster_members {
|
|
my $ip = [];
|
|
foreach my $line (qx(corosync-cmapctl)){
|
|
push @{$ip}, $1 if ($line =~ m/ip\((\d+\.\d+\.\d+\.\d+)\)/);
|
|
}
|
|
return $ip;
|
|
}
|
|
|
|
sub redirect_ipfo {
|
|
my $ip = shift;
|
|
print "Redirecting failover IP " . join(', ', @{$ip}) . "\n";
|
|
my $response = $lwp->post("https://api.online.net/api/v1/server/failover/edit",
|
|
Authorization => "Authorization: Bearer $conf->{online_api}",
|
|
Content => {
|
|
"source" => join(',', @{$ip}),
|
|
"destination" => get_public_ip()
|
|
}
|
|
);
|
|
unless ($response->is_success){
|
|
die "an error occured while querying the API" .
|
|
"The error is: " . $response->status_line;
|
|
}
|
|
}
|
|
|
|
# Get a VM configuration
|
|
sub get_guest_conf {
|
|
if ($ENV{PVE_GUEST_TYPE} eq 'lxc'){
|
|
return PVE::LXC::Config->load_config(shift);
|
|
} else {
|
|
return PVE::QemuConfig->load_config(shift);
|
|
}
|
|
}
|
|
|
|
# Unplug and plug back nics connected on the WAN bridge
|
|
# Needed when moving a failover IP to force flushing the ARP cache
|
|
# in the guest as the gateway doesn't change, but its MAC does
|
|
sub set_guest_nic_down_up {
|
|
my $vmid = shift;
|
|
my $vm_conf = get_guest_conf($vmid);
|
|
my $nics = [];
|
|
my @mod_nic = ();
|
|
foreach my $key (keys %{$vm_conf}){
|
|
# Only process netX elements, and which are connected on the WAN bridge
|
|
next unless ($key =~ m/^net\d+/ && $vm_conf->{$key} =~ m/bridge=$conf->{wan_bridge}/);
|
|
my $props = $vm_conf->{$key};
|
|
next if ($props =~ m/link_down=0/);
|
|
push @mod_nic, $key;
|
|
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
|
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $key off");
|
|
} else {
|
|
# PVE 5.x doesn't have PVE::QemuServer::Monitor::hmp_cmd
|
|
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $key off");
|
|
}
|
|
}
|
|
sleep 1;
|
|
foreach my $nic (@mod_nic){
|
|
if (defined &PVE::QemuServer::Monitor::hmp_cmd){
|
|
PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $nic on");
|
|
} else {
|
|
PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $nic on");
|
|
}
|
|
}
|
|
}
|
|
|