ansible-roles/roles/pve/files/pve-online

#!/usr/bin/perl -w

use strict;
use warnings;
use Getopt::Long;
use Config::Simple;
use LWP::UserAgent;
use JSON;
use PVE::QemuServer;
use PVE::LXC::Config;
use Array::Diff;
use Net::Address::IP::Local;
use Net::Route::Table;
use NetAddr::IP;
use Data::Validate::IP qw(is_ipv4);
use Data::Dumper;

$| = 1;

my $config = '/etc/pve-online.conf';
my ($update_routes, $update_gre, $migrate_ipfo) = undef;

GetOptions(
  "config=s"       => \$config,
  "update-routes"  => \$update_routes,
  "update-gre"     => \$update_gre,
  "migrate-ipfo=i" => \$migrate_ipfo
);

# Config can be stored in /etc/pve
# Lets wait a bit for it to be available
if ($config =~ m|^/etc/pve|){
  my $t = 0;
  while ($t < 120){
    if (!-e $config){
      print "$config not yet available\n";
      sleep 2;
      $t += 2;
    } else {
      last;
    }
  }
}
if (!-e $config){
  die "$config doesn't exist\n";
}

my $cfg = new Config::Simple;
$cfg->read($config);
my $conf = $cfg->get_block('general');
my $lwp = new LWP::UserAgent;
my $online_id = undef;

if (!$conf){
  die "No general section found in $config\n";
}
if (!$conf->{online_api}){
  die "No online_api defined in $config\n";
}

# Set some defaults
$conf->{wan_bridge}  ||= 'vmbr1';
$conf->{migrate_flush_arp} ||= 'yes';

if ($update_routes){
  update_routes();
}
if ($update_gre){
  if (defined $conf->{vlan_bridge}){
    update_gre();
  } else {
    print "No VLAN bridge defined. Don't setup GRE tunnels\n";
  }
}
if ($migrate_ipfo){
  my $ipfo = [];
  # We parse the description field which contains the list of ipfo
  # attached to this VM
  foreach my $line ( split("\n", get_guest_conf($migrate_ipfo)->{description} || "") ){
    if ($line =~ m/^\s*ipfo\d*:\s+(\d+\.\d+\.\d+\.\d+)/){
      my $candidate = $1;
      if (is_ipv4($candidate)){
        push @{$ipfo}, $candidate;
        print "Found IP $candidate assigned to guest $migrate_ipfo\n";
      } else {
        print "Found $candidate assigned to guest $migrate_ipfo which doesn't look like a valid IP\n";
      }
    }
  }
  $online_id ||= get_online_id();
  # Now we check if we need to migrate IPFO on the local server
  my $ipfo_diff = Array::Diff->diff(get_ip_fo($online_id), $ipfo);
  if (scalar @{$ipfo_diff->added} > 0){
    print "Update needed. " . join(' ', @{$ipfo_diff->added}) . " should be redirected on server $online_id\n";
    redirect_ipfo($ipfo_diff->added);
    update_routes();
    if ( $ENV{PVE_GUEST_TYPE} ne 'lxc' and $conf->{migrate_flush_arp} =~ m/^1|yes|true|on$/i ){
      set_guest_nic_down_up($migrate_ipfo);
    }
  }
}

#####################################
# Sub routines
#####################################

# Query Online's API
sub query_online_api{
  my $uri = shift;
  $uri = '/api/v1' . $uri if ($uri !~ m|^/api/v1|);
  my $response = $lwp->get("https://api.online.net$uri",
    "Authorization" => "Authorization: Bearer $conf->{online_api}",
  );
  unless ($response->is_success){
    die "an error occured while querying the API" .
        "The error is: " . $response->status_line;
  }
  return from_json($response->content);
}

# Update routes for ARP Proxy
sub update_routes {
  $online_id ||= get_online_id();
  # This is the list of IP which we should have
  # on the routing table
  my $routes_online = get_ip_fo($online_id);

  # This is the actual list of IP for which we have routes
  my $routes_local = get_local_routes();

  # Now, we have to remove routes for those in $routes_local but not in $routes_online
  # And add routes for those in $routes_online but not in $routes_local
  my $diff = Array::Diff->diff($routes_online, $routes_local);
  foreach my $route (@{$diff->added}){
    next if (grep { $_ eq $route } @{$diff->deleted});
    print "Removing route for $route\n";
    system(
      '/sbin/ip',
      'route',
      'del',
      $route,
      'dev',
      $conf->{wan_bridge}
    );
  }
  foreach my $route (@{$diff->deleted}){
    next if (grep { $_ eq $route } @{$diff->added});
    print "Adding route for $route\n";
    system(
      '/sbin/ip',
      'route',
      'add',
      $route . '/32',
      'dev',
      $conf->{wan_bridge}
    );
  }
}

# Get the list of routes defined on $conf->{wan_bridge}
sub get_local_routes {
  my $ip = [];
  my $routes = Net::Route::Table->from_system();
  foreach my $route (@{$routes->all_routes()}){
    if ($route->{interface} eq $conf->{wan_bridge} and $route->destination()->masklen() == 32){
      push @{$ip}, $route->destination()->addr();
    }
  }
  return $ip;
}

# Get the list of IP failover assigned to a server. Taks a server ID as only arg
sub get_ip_fo {
  my $srv_id = shift;
  return get_srv_info($srv_id)->{network}->{ipfo};
}

# Return server info
sub get_srv_info {
  my $srv_id = shift;
  return query_online_api('/server/info/' . $srv_id);
}

# Return this server's public IP
sub get_public_ip {
  return Net::Address::IP::Local->public_ipv4;
}

# Return Online's server id
sub get_online_id {
  if (-e '/tmp/online_id'){
    open my $id_file, '</tmp/online_id';
    $online_id = <$id_file>;
    close $id_file;
  } else {
    my $ip = get_public_ip();
    foreach my $srv (@{query_online_api('/server')}){
      my $info = query_online_api($srv);
      if ($info->{network}->{ip}[0] eq $ip){
        $online_id = $info->{id};
        last;
      }
    }
    open my $id_file, ">/tmp/online_id";
    print $id_file $online_id;
    close $id_file;
  }
  print "My Online's ID is $online_id\n";
  return $online_id;
}

sub update_gre {
  # We have to setup GRE tunnels with all other members
  # to connect $conf->{vlan_bridge} between every nodes
  # Something like
  # ovs-vsctl add-port vmbr1 gre0 -- set interface gre0 type=gre options:remote_ip=''10.29.254.2''
  # We just have to automate this for every nodes of the cluster
  print "Getting cluster status...\n";
  my $members = get_cluster_members();
  print "Found " . scalar @{$members} . " members\n";
  my $gre = 0;

  print "Counting GRE ports...\n";
  my @ports = qx(ovs-vsctl list-ports $conf->{vlan_bridge});
  my @gre_ports = grep { $_ =~ m/^gre/ } @ports;
  print "Found " . scalar @gre_ports . " GRE ports\n";

  if (scalar @gre_ports ne scalar @{$members} - 1){
    print "We need to update GRE tunnels\n";
    # Remove all greX ports from the VLAN bridge
    foreach my $port ( @ports ){
      chomp($port);
      next unless ($port =~ m/^gre\d+$/);
      print "Removing port $port from $conf->{vlan_bridge}\n";
      system(
        'ovs-vsctl',
        'del-port',
        $conf->{vlan_bridge},
        $port
      );
    }
    # And setup one GRE tunnel per node
    foreach my $member (@{$members}){
      # We must skip our own node
      if (Net::Address::IP::Local->connected_to($member) ne $member){
        print "Adding GRE interface gre$gre to tunnel with $member\n";
        system(
          'ovs-vsctl',
          'add-port',
          $conf->{vlan_bridge},
          'gre' . $gre,
          '--',
          'set',
          'interface',
          'gre' . $gre,
          'type=gre',
          'options:remote_ip=' . $member
        );
        $gre++;
      }
    }
  }
}

# Get the list of members of this proxmox cluster
sub get_cluster_members {
  my $ip = [];
  foreach my $line (qx(corosync-cmapctl)){
    push @{$ip}, $1 if ($line =~ m/ip\((\d+\.\d+\.\d+\.\d+)\)/);
  }
  return $ip;
}

sub redirect_ipfo {
  my $ip = shift;
  print "Redirecting failover IP " . join(', ', @{$ip}) . "\n";
  my $response = $lwp->post("https://api.online.net/api/v1/server/failover/edit",
    Authorization => "Authorization: Bearer $conf->{online_api}",
    Content => {
      "source"        => join(',', @{$ip}),
      "destination"   => get_public_ip()
    }
  );
  unless ($response->is_success){
    die "an error occured while querying the API" .
        "The error is: " . $response->status_line;
  }
}

# Get a VM configuration
sub get_guest_conf {
  if ($ENV{PVE_GUEST_TYPE} eq 'lxc'){
    return PVE::LXC::Config->load_config(shift);
  } else {
    return PVE::QemuConfig->load_config(shift);
  }
}

# Unplug and plug back nics connected on the WAN bridge
# Needed when moving a failover IP to force flushing the ARP cache
# in the guest as the gateway doesn't change, but its MAC does
sub set_guest_nic_down_up {
  my $vmid = shift;
  my $vm_conf = get_guest_conf($vmid);
  my $nics = [];
  my @mod_nic = ();
  foreach my $key (keys %{$vm_conf}){
    # Only process netX elements, and which are connected on the WAN bridge
    next unless ($key =~ m/^net\d+/ && $vm_conf->{$key} =~ m/bridge=$conf->{wan_bridge}/);
    my $props = $vm_conf->{$key};
    next if ($props =~ m/link_down=0/);
    push @mod_nic, $key;
    if (defined &PVE::QemuServer::Monitor::hmp_cmd){
      PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $key off");
    } else {
      # PVE 5.x doesn't have PVE::QemuServer::Monitor::hmp_cmd
      PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $key off");
    }
  }
  sleep 1;
  foreach my $nic (@mod_nic){
    if (defined &PVE::QemuServer::Monitor::hmp_cmd){
      PVE::QemuServer::Monitor::hmp_cmd($vmid, "set_link $nic on");
    } else {
      PVE::QemuServer::vm_human_monitor_command($vmid, "set_link $nic on");
    }
  }
}