Improved logging of removal of old backups

Added option to dry-run removal of old backups
Updated to use the new env variable targetfile instead of tarfile
jobFolder is now automatically created if needed
Rclone now has hardcoded shell and home due to changes in Proxmox7

Author: farthinder

ProxmoxEncryptedBackup.py

@@ -117,9 +117,11 @@ def encryptAndUploadFiles(recipient, sourceDirectory, sourceFiles, destinationFo
 
 def cleanupJobFiles():
 
+
     if Settings.rcloneRemoveOldBackups:
+        logging.info("Preparing to remove old backups files from:" + Settings.rcloneRemoteName)
         rclone = Rclone.Rclone()
-        rclone.removeOldFiles(Settings.rcloneRemoveThreshold,Settings.rcloneRemoteName)
+        rclone.removeOldFiles(Settings.rcloneRemoveThreshold,Settings.rcloneRemoteName, dryRun=Settings.rcloneRemoveOldBackupsDryRun)
 
     logging.info("Preparing to remove old job files from:" + Settings.jobFolder)
 
@@ -150,13 +152,13 @@ def cleanupJobFiles():
     threadFutures = []
     threadPool = ThreadPoolExecutor(max_workers=Settings.threads)
     for index in filesToEncrypt:
-        outputFilename = re.search("vzdump-(?:qemu|lxc)-(.*)\.\w*?$", index["tarfile"])[1]
+        outputFilename = re.search("vzdump-(?:qemu|lxc)-(.*)\.\w*?$", index["targetfile"])[1]
         outputFilename += ".enc"
         threadFutures.append(
             threadPool.submit(encryptAndUploadFiles,
                               recipient=Settings.gpgRecipient,
                               sourceDirectory=pathToFiles,
-                              sourceFiles=[index["tarfile"], index["logfile"]],
+                              sourceFiles=[index["targetfile"], index["logfile"]],
                               destinationFolder=Settings.gpgOutputDirectory,
                               destinationFileName=outputFilename,
                               remoteName=Settings.rcloneRemoteName
@@ -176,3 +178,4 @@ def cleanupJobFiles():
 
 
     cleanupJobFiles()
+

ProxmoxEncryptedBackupSettings.py

@@ -27,4 +27,5 @@
 rcloneVerifyUploads = True  # If True, after the upload the local and remote files will have their hashes compared
 rcloneRemoveSourceFile = True  # If True, the upload source file will be deleted after upload. Note this is the .enc file and not the original backup files.
 rcloneRemoveOldBackups = False # If True, old remote files will be deleted
+rcloneRemoveOldBackupsDryRun = True #If True AND rcloneRemoveOldBackups is True, will only list files that would be deleted
 rcloneRemoveThreshold = "14d" # Backups older than this threshold are deleted. Format: ms|s|m|h|d|w|M|y. Example: 14d = 14 days

ProxmoxEventHandler.py

@@ -23,6 +23,11 @@ def __init__(self, jobFolder, jobId = None):
         logging.info("Job File:" + self.jobFilePath)
         logging.debug("Raw Input parameters:" + ",".join(sys.argv))
 
+
+        if not os.path.exists(jobFolder):
+            logging.debug("Job folder does not already exist, creating it.")
+            os.makedirs(jobFolder)
+
         parser = argparse.ArgumentParser()
         parser.add_argument("phase", help="backup phase")
         parser.add_argument("attributes", help="backup attributes mode and vmid", nargs='*')
@@ -149,8 +154,8 @@ def getPhase(self):
             dumpdir = os.environ["DUMPDIR"]
             storeid = os.environ["STOREID"]
             hostname = os.environ["HOSTNAME"]
-            # tarfile is only available in phase 'backup-end'
-            tarfile = os.environ["TARFILE"]
+            # TARGET is only available in phase 'backup-end'
+            targetfile = os.environ["TARGET"] #Contains the full path to the current VM backup file
             # logfile is only available in phase 'log-end'
             logfile = os.environ["LOGFILE"]
 
@@ -171,7 +176,7 @@ def getPhase(self):
                 'dumpdir': dumpdir,
                 'storeid': storeid,
                 'hostname': hostname,
-                'tarfile': tarfile,
+                'targetfile': targetfile,
                 'logfile': logfile,
                 'mode': mode
             }
@@ -189,21 +194,21 @@ def getPhase(self):
 
     '''
 
-    Returns a an array with dics with quoted tarfile and logfile elements:
+    Returns a an array with dics with quoted targetfile and logfile elements:
         [
-            {tarfile: "vzdump-qemu-107-2019_12_29-23_22_54.vma",logfile: "vzdump-qemu-107-2019_12_29-23_22_54.log"},
-            {tarfile: "vzdump-qemu-108-2019_12_29-23_22_54.vma",logfile: "vzdump-qemu-108-2019_12_29-23_22_54.log"},
+            {targetfile: "vzdump-qemu-107-2019_12_29-23_22_54.vma",logfile: "vzdump-qemu-107-2019_12_29-23_22_54.log"},
+            {targetfile: "vzdump-qemu-108-2019_12_29-23_22_54.vma",logfile: "vzdump-qemu-108-2019_12_29-23_22_54.log"},
         ]
     '''
     def getFilesFromPhase(self, phase):
         logging.debug("Gathering files from phase: " + phase)
         filePaths = []
 
         for x in self.jobinfo[phase]:
-            tarfile = self.jobinfo[phase][x]["tarfile"].split("/")[-1]
+            targetfile = self.jobinfo[phase][x]["targetfile"].split("/")[-1]
             logfile =  self.jobinfo[phase][x]["logfile"].split("/")[-1]
 
-            filePaths.append({"tarfile" : tarfile, "logfile" : logfile})
+            filePaths.append({"targetfile" : targetfile, "logfile" : logfile})
 
         logging.debug("Got " + str(len(filePaths) * 2) + " files")
 

Rclone.py

@@ -13,7 +13,7 @@ class Rclone:
     def __init__(self, rcloneLogFilePath=None):
 
         self.rcloneLogFilePath = rcloneLogFilePath
-        self.bash = pexpect.spawn("/bin/bash", echo=False)
+        self.bash = pexpect.spawn("/bin/bash", echo=False, env = {"SHELL": "/bin/bash", "HOME": "/root"})
 
         if self.rcloneLogFilePath is not None:
             self.bash.logfile = open(rcloneLogFilePath, 'wb')
@@ -35,7 +35,6 @@ def transferSingleFileToRemoteRoot(self, rcloneSourceFile, rcloneRemote, verifyU
         rcloneSourceFileName = '"' + rcloneSourceFileName + '"'
 
         logging.info("\tStarting transfer of " + rcloneSourceFile + " to " + rcloneRemote + rcloneSourceFileName)
-
         self.bash.sendline(
             f"rclone copyto {rcloneSourceFile} {rcloneRemote}{rcloneSourceFileName} -P ; echo RCLONE FINISHED WITH STATUS $?")
 
@@ -91,15 +90,24 @@ def transferSingleFileToRemoteRoot(self, rcloneSourceFile, rcloneRemote, verifyU
             elif result == 2:
                 logging.error("\tGdrive upload timed out")
                 sys.exit(1)
-
-    def removeOldFiles(self, tresholdtime, rcloneRemote):
-        logging.info("\tDelete remote files older then " + tresholdtime)
-        self.bash.sendline(f"rclone delete --min-age {tresholdtime} {rcloneRemote}; echo DELETEDONE")
+                
+    def removeOldFiles(self, tresholdtime, rcloneRemote, dryRun:bool = True):
+        
+        if dryRun:
+            logging.info("\tRunning dry run of deleting remote files older then " + tresholdtime)
+            logging.info(f"rclone delete --dry-run --min-age {tresholdtime} {rcloneRemote}; echo DELETEDONE")
+            self.bash.sendline(f"rclone delete --dry-run --min-age {tresholdtime} {rcloneRemote}; echo DELETEDONE")
+        else:
+            logging.info("\tDeleting  remote files older then " + tresholdtime)
+            self.bash.sendline(f"rclone delete --min-age {tresholdtime} {rcloneRemote}; echo DELETEDONE")
 
         result = self.bash.expect(["DELETEDONE", pexpect.TIMEOUT])
 
+        deleteOutput = self.bash.before.decode("utf-8")
+        logging.debug(deleteOutput)
+
         if result == 0:
-            logging.info("\t Remote files were removed")
+            logging.info("\t Remote files were removed (Dry Run: " + str(dryRun) + ")")
             return True
         else:
             bashOutput = self.bash.read_nonblocking(size=500, timeout=1)