#!/usr/bin/env python3 # Copyright (c) 2016, Antonio SJ Musumeci # # Permission to use, copy, modify, and/or distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import argparse import ctypes import errno import fnmatch import os import subprocess import sys _libc = ctypes.CDLL("libc.so.6",use_errno=True) _lgetxattr = _libc.lgetxattr _lgetxattr.argtypes = [ctypes.c_char_p,ctypes.c_char_p,ctypes.c_void_p,ctypes.c_size_t] def lgetxattr(path,name): if type(path) == str: path = path.encode('utf-8','surrogateescape') if type(name) == str: name = name.encode('utf-8','surrogateescape') length = 64 while True: buf = ctypes.create_string_buffer(length) res = _lgetxattr(path,name,buf,ctypes.c_size_t(length)) if res >= 0: return buf.raw[0:res] else: err = ctypes.get_errno() if err == errno.ERANGE: length *= 2 elif err == errno.ENODATA: return None else: raise IOError(err,os.strerror(err),path) def ismergerfs(path): try: lgetxattr(path,'user.mergerfs.version') return True except IOError as e: return False def mergerfs_control_file(basedir): ctrlfile = os.path.join(basedir,'.mergerfs') if os.path.exists(ctrlfile): return ctrlfile else: dirname = os.path.dirname(basedir) return mergerfs_control_file(dirname) def mergerfs_srcmounts(ctrlfile): srcmounts = lgetxattr(ctrlfile,'user.mergerfs.srcmounts') srcmounts = srcmounts.decode().split(':') return srcmounts def match(filename,matches): for match in matches: if fnmatch.fnmatch(filename,match): return True return False def exclude_by_size(filepath,exclude_lt,exclude_gt): try: st = os.lstat(filepath) if exclude_lt and st.st_size < exclude_lt: return True if exclude_gt and st.st_size > exclude_gt: return True return False except: return False def find_a_file(src, relpath, file_includes,file_excludes, path_includes,path_excludes, exclude_lt,exclude_gt): basepath = os.path.join(src,relpath) for (dirpath,dirnames,filenames) in os.walk(basepath): for filename in filenames: filepath = os.path.join(dirpath,filename) if match(filename,file_excludes): continue if match(filepath,path_excludes): continue if not match(filename,file_includes): continue if not match(filepath,path_includes): continue if exclude_by_size(filepath,exclude_lt,exclude_gt): continue return os.path.relpath(filepath,src) return None def execute(args): return subprocess.call(args) def move_file(src,dst,relfile): frompath = os.path.join(src,'./',relfile) topath = dst+'/' args = ['rsync', '-alXA', '--relative', '--progress', '--remove-source-files', frompath, topath] return execute(args) def freespace_percentage(srcmounts): lfsp = [] for srcmount in srcmounts: vfs = os.statvfs(srcmount) avail = vfs.f_bavail * vfs.f_frsize total = vfs.f_blocks * vfs.f_frsize per = avail / total lfsp.append((srcmount,per)) return sorted(lfsp, key=lambda x: x[1]) def all_within_range(l,n): if len(l) == 0 or len(l) == 1: return True return (abs(l[0][1] - l[-1][1]) <= n) def human_to_bytes(s): m = s[-1] if m == 'K': i = int(s[0:-1]) * 1024 elif m == 'M': i = int(s[0:-1]) * 1024 * 1024 elif m == 'G': i = int(s[0:-1]) * 1024 * 1024 * 1024 elif m == 'T': i = int(s[0:-1]) * 1024 * 1024 * 1024 * 1024 else: i = int(s) return i def buildargparser(): parser = argparse.ArgumentParser(description='balance files on a mergerfs mount based on percentage drive filled') parser.add_argument('dir', type=str, help='starting directory') parser.add_argument('-p', dest='percentage', type=float, default=2.0, help='percentage range of freespace (default 2.0)') parser.add_argument('-i','--include', dest='include', type=str, action='append', default=[], help='fnmatch compatible file filter (can use multiple times)') parser.add_argument('-e','--exclude', dest='exclude', type=str, action='append', default=[], help='fnmatch compatible file filter (can use multiple times)') parser.add_argument('-I','--include-path', dest='includepath', type=str, action='append', default=[], help='fnmatch compatible path filter (can use multiple times)') parser.add_argument('-E','--exclude-path', dest='excludepath', type=str, action='append', default=[], help='fnmatch compatible path filter (can use multiple times)') parser.add_argument('-s', dest='excludelt', type=str, default='0', help='exclude files smaller than [KMGT] bytes') parser.add_argument('-S', dest='excludegt', type=str, default='0', help='exclude files larger than [KMGT] bytes') return parser def main(): parser = buildargparser() args = parser.parse_args() args.dir = os.path.realpath(args.dir) ctrlfile = mergerfs_control_file(args.dir) if not ismergerfs(ctrlfile): print("%s is not a mergerfs mount" % args.dir) sys.exit(1) relpath = '' mntpoint = os.path.dirname(ctrlfile) if args.dir != mntpoint: relpath = os.path.relpath(args.dir,mntpoint) file_includes = ['*'] if not args.include else args.include file_excludes = args.exclude path_includes = ['*'] if not args.includepath else args.includepath path_excludes = args.excludepath exclude_lt = human_to_bytes(args.excludelt) exclude_gt = human_to_bytes(args.excludegt) srcmounts = mergerfs_srcmounts(ctrlfile) percentage = args.percentage / 100 seen = set() try: l = freespace_percentage(srcmounts) while not all_within_range(l,percentage): todrive = l[-1][0] relfilepath = None while not relfilepath: fromdrive = l[0][0] del l[0] relfilepath = find_a_file(fromdrive, relpath, file_includes,file_excludes, path_includes,path_excludes, exclude_lt,exclude_gt) if fromdrive == todrive: break if relfilepath in seen: break; seen.add(relfilepath) print('from: {}\nto: {}'.format(fromdrive,todrive)) rv = move_file(fromdrive,todrive,relfilepath) if rv: break; l = freespace_percentage(srcmounts) except KeyboardInterrupt: print("exiting: CTRL-C pressed") sys.exit(0) if __name__ == "__main__": main()