commit b8b945187ec6a7928a05c72f064c1279a008a0c9 Author: Daniel Berteaud Date: Tue May 15 23:02:28 2012 +0200 Initial import from https://gist.github.com/1338263 diff --git a/blocksync.py b/blocksync.py new file mode 100644 index 0000000..9235d3b --- /dev/null +++ b/blocksync.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +""" +Synchronise block devices over the network + +Copyright 2006-2008 Justin Azoff +Copyright 2011 Robert Coup +License: GPL + +Getting started: + +* Copy blocksync.py to the home directory on the remote host +* Make sure your remote user can either sudo or is root itself. +* Make sure your local user can ssh to the remote host +* Invoke: +sudo python blocksync.py /dev/source user@remotehost /dev/dest +""" + +import sys +from zlib import adler32 +import subprocess +import time + +SAME = "same\n" +DIFF = "diff\n" + + +def do_open(f, mode): + f = open(f, mode) + f.seek(0, 2) + size = f.tell() + f.seek(0) + return f, size + + +def getblocks(f, blocksize): + while 1: + block = f.read(blocksize) + if not block: + break + yield block + + +def server(dev, blocksize): + print dev, blocksize + f, size = do_open(dev, 'r+') + print size + sys.stdout.flush() + + for block in getblocks(f, blocksize): + print "%08x" % (adler32(block) & 0xFFFFFFFF) + sys.stdout.flush() + res = sys.stdin.readline() + if res != SAME: + newblock = sys.stdin.read(blocksize) + f.seek(-blocksize, 1) + f.write(newblock) + + +def sync(srcdev, dsthost, dstdev=None, blocksize=1024 * 1024): + + if not dstdev: + dstdev = srcdev + + print "Block size is %0.1f MB" % (float(blocksize) / (1024 * 1024)) + cmd = ['ssh', '-c', 'blowfish', dsthost, 'sudo', 'python', 'blocksync.py', 'server', dstdev, '-b', str(blocksize)] + print "Running: %s" % " ".join(cmd) + + p = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) + p_in, p_out = p.stdin, p.stdout + + line = p_out.readline() + p.poll() + if p.returncode is not None: + print "Error connecting to or invoking blocksync on the remote host!" + sys.exit(1) + + a, b = line.split() + if a != dstdev: + print "Dest device (%s) doesn't match with the remote host (%s)!" % (dstdev, a) + sys.exit(1) + if int(b) != blocksize: + print "Source block size (%d) doesn't match with the remote host (%d)!" % (blocksize, int(b)) + sys.exit(1) + + try: + f, size = do_open(srcdev, 'r') + except Exception, e: + print "Error accessing source device! %s" % e + sys.exit(1) + + line = p_out.readline() + p.poll() + if p.returncode is not None: + print "Error accessing device on remote host!" + sys.exit(1) + remote_size = int(line) + if size != remote_size: + print "Source device size (%d) doesn't match remote device size (%d)!" % (size, remote_size) + sys.exit(1) + + same_blocks = diff_blocks = 0 + + print "Starting sync..." + t0 = time.time() + t_last = t0 + for i, l_block in enumerate(getblocks(f, blocksize)): + l_sum = "%08x" % (adler32(l_block) & 0xFFFFFFFF) + r_sum = p_out.readline().strip() + + if l_sum == r_sum: + p_in.write(SAME) + p_in.flush() + same_blocks += 1 + else: + p_in.write(DIFF) + p_in.flush() + p_in.write(l_block) + p_in.flush() + diff_blocks += 1 + + t1 = time.time() + if t1 - t_last > 1: + rate = (i + 1.0) * blocksize / (1024.0 * 1024.0) / (t1 - t0) + print "\rsame: %d, diff: %d, %d/%d, %5.1f MB/s" % (same_blocks, diff_blocks, same_blocks + diff_blocks, size / blocksize, rate), + t_last = t1 + + print "\n\nCompleted in %d seconds" % (time.time() - t0) + + return same_blocks, diff_blocks + +if __name__ == "__main__": + from optparse import OptionParser + parser = OptionParser(usage="%prog [options] /dev/source user@remotehost [/dev/dest]") + parser.add_option("-b", "--blocksize", dest="blocksize", action="store", type="int", help="block size (bytes)", default=1024 * 1024) + (options, args) = parser.parse_args() + + if len(args) < 2: + parser.print_help() + print __doc__ + sys.exit(1) + + if args[0] == 'server': + dstdev = args[1] + server(dstdev, options.blocksize) + else: + srcdev = args[0] + dsthost = args[1] + if len(args) > 2: + dstdev = args[2] + else: + dstdev = None + sync(srcdev, dsthost, dstdev, options.blocksize) + +