Merge pull request #54 from robweber/zip_files

Zip files

README.txt

@@ -8,6 +8,8 @@ Remote Destination/File Selection:
 
 In the addon settings you can define a remote path for the destination of your xbmc files. Each backup will create a folder named in a YYYYMMDDHHmm format so you can create multiple backups. You can keep a set number of backups by setting the integer value of the Backups to Keep setting greater than 0. 
 
+If you choose to compress your backups there are a few things you need to be aware of. Compressing takes place on the server you are trying to backup and then only the archive is copied to the remote backup location. This means you must have sufficient space available to allow for creating the archive. When restoring a zipped archive the process is the same. It is first copied to your local storage, extracted, and the contents put to their correct locations. The archive is then deleted. Zipped and non-zipped backups can be mixed in the same archive folder.  
+
 On the Backup Selection page you can select which items from your user profile folder will be sent to the backup location. By default all are turned on except the Addon Data directory. 
 
 You can also define non-XBMC directories on your device. See "Custom Directories" for more information on how these are handled. 

addon.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <addon id="script.xbmcbackup"
-    name="XBMC Backup" version="0.5.6" provider-name="robweber">
+    name="XBMC Backup" version="0.5.7" provider-name="robweber">
   <requires>
     <import addon="xbmc.python" version="2.14.0"/>
   </requires>

changelog.txt

@@ -1,3 +1,7 @@
+Version 0.5.7
+
+added option to compress backups, uses local source for staging the zip before sending to remote
+
 Version 0.5.6
 
 fix dropbox delete recursion error - thanks durd

resources/lib/backup.py

@@ -2,10 +2,9 @@ import xbmc
 import xbmcgui
 import xbmcvfs
 import utils as utils
-import os.path
 import time
 import json
-from vfs import XBMCFileSystem,DropboxFileSystem
+from vfs import XBMCFileSystem,DropboxFileSystem,ZipFileSystem
 
 def folderSort(aKey):
     result = aKey[0]
@@ -21,9 +20,11 @@ class XbmcBackup:
     Backup = 0
     Restore = 1
 
-    #remote file system
+    #file systems
     xbmc_vfs = None
     remote_vfs = None
+    saved_remote_vfs = None
+    
     restoreFile = None
     remote_base_path = None
     
@@ -80,6 +81,21 @@ class XbmcBackup:
 
                 result.append((aDir,folderName))
 
+        for aFile in files:
+            file_ext = aFile.split('.')[-1]
+           
+            if(file_ext == 'zip'):
+                
+                #folder may or may not contain time, older versions didn't include this
+                folderName = ''
+                if(len(aFile ) > 8):
+                    folderName = aFile [6:8] + '-' + aFile [4:6] + '-' + aFile [0:4] + " " + aFile [8:10] + ":" + aFile [10:12]
+                else:
+                    folderName = aFile [6:8] + '-' + aFile [4:6] + '-' + aFile [0:4]
+
+                result.append((aFile ,folderName))
+                
+
         result.sort(key=folderSort)
         
         return result
@@ -98,9 +114,15 @@ class XbmcBackup:
         #append backup folder name
         progressBarTitle = utils.getString(30010) + " - "
         if(mode == self.Backup and self.remote_vfs.root_path != ''):
+            if(utils.getSetting("compress_backups") == 'true'):
+                #save the remote file system and use the zip vfs
+                self.saved_remote_vfs = self.remote_vfs
+                self.remote_vfs = ZipFileSystem(xbmc.translatePath("special://temp/xbmc_backup_temp.zip"),"w")
+                
             self.remote_vfs.set_root(self.remote_vfs.root_path + time.strftime("%Y%m%d%H%M") + "/")
             progressBarTitle = progressBarTitle + utils.getString(30016)
         elif(mode == self.Restore and self.restore_point != None and self.remote_vfs.root_path != ''):
+            if(self.restore_point.split('.')[-1] != 'zip'):
                 self.remote_vfs.set_root(self.remote_vfs.root_path + self.restore_point + "/")
             progressBarTitle = progressBarTitle + utils.getString(30017)
         else:
@@ -135,36 +157,36 @@ class XbmcBackup:
          
             #go through each of the user selected items and write them to the backup store
             if(utils.getSetting('backup_addons') == 'true'):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "addons")
+                fileManager.addFile("-addons")
                 fileManager.walkTree(xbmc.translatePath('special://home/addons'))
 
-            self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata")
+            fileManager.addFile("-userdata")
 
             if(utils.getSetting('backup_addon_data') == 'true'):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/addon_data")
+                fileManager.addFile("-userdata/addon_data")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/addon_data'))
 
             if(utils.getSetting('backup_database') == 'true'):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/Database")
+                fileManager.addFile("-userdata/Database")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/Database'))
         
             if(utils.getSetting("backup_playlists") == 'true'):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/playlists")
+                fileManager.addFile("-userdata/playlists")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/playlists'))
 
             if(utils.getSetting('backup_profiles') == 'true'):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/profiles")
+                fileManager.addFile("-userdata/profiles")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/profiles'))
             
             if(utils.getSetting("backup_thumbnails") == "true"):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/Thumbnails")
+                fileManager.addFile("-userdata/Thumbnails")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/Thumbnails'))
  
             if(utils.getSetting("backup_config") == "true"):
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/keymaps")
+                fileManager.addFile("-userdata/keymaps")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/keymaps'))
                 
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/peripheral_data")
+                fileManager.addFile("-userdata/peripheral_data")
                 fileManager.walkTree(xbmc.translatePath('special://home/userdata/peripheral_data'))
             
                 #this part is an oddity
@@ -177,12 +199,14 @@ class XbmcBackup:
             self.filesTotal = fileManager.size()
             allFiles.append({"source":self.xbmc_vfs.root_path,"dest":self.remote_vfs.root_path,"files":fileManager.getFiles()})
 
+            orig_base_path = self.remote_vfs.root_path
+            
             #check if there are custom directories
             if(utils.getSetting('custom_dir_1_enable') == 'true' and utils.getSetting('backup_custom_dir_1') != ''):
 
                 #create a special remote path with hash                
                 self.xbmc_vfs.set_root(utils.getSetting('backup_custom_dir_1'))
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "custom_" + self._createCRC(self.xbmc_vfs.root_path))
+                fileManager.addFile("-custom_" + self._createCRC(self.xbmc_vfs.root_path))
 
                 #walk the directory
                 fileManager.walkTree(self.xbmc_vfs.root_path)
@@ -193,7 +217,7 @@ class XbmcBackup:
 
                 #create a special remote path with hash                
                 self.xbmc_vfs.set_root(utils.getSetting('backup_custom_dir_2'))
-                self.remote_vfs.mkdir(self.remote_vfs.root_path + "custom_" + self._createCRC(self.xbmc_vfs.root_path))
+                fileManager.addFile("-custom_" + self._createCRC(self.xbmc_vfs.root_path))
 
                 #walk the directory
                 fileManager.walkTree(self.xbmc_vfs.root_path)
@@ -208,12 +232,60 @@ class XbmcBackup:
                 self.remote_vfs.set_root(fileGroup['dest'])
                 self.backupFiles(fileGroup['files'],self.xbmc_vfs,self.remote_vfs)
             
+            #reset remote and xbmc vfs
+            self.xbmc_vfs.set_root(xbmc.translatePath("special://home"))
+            self.remote_vfs.set_root(orig_base_path)
+
+            if(utils.getSetting("compress_backups") == 'true'):
+                #send the zip file to the real remote vfs
+                zip_name = self.remote_vfs.root_path[:-1] + ".zip"
+                self.remote_vfs.cleanup()
+                self.xbmc_vfs.rename(xbmc.translatePath("special://temp/xbmc_backup_temp.zip"), xbmc.translatePath("special://temp/" + zip_name))
+                fileManager.addFile(xbmc.translatePath("special://temp/" + zip_name))
+               
+                #set root to data dir home 
+                self.xbmc_vfs.set_root(xbmc.translatePath("special://temp/"))
+               
+                self.remote_vfs = self.saved_remote_vfs
+                self.progressBar.updateProgress(98, "Copying Zip Archive")
+                self.backupFiles(fileManager.getFiles(),self.xbmc_vfs, self.remote_vfs)
+                
+                #delete the temp zip file
+                self.xbmc_vfs.rmfile(xbmc.translatePath("special://temp/" + zip_name))
+
             #remove old backups
             self._rotateBackups()
 
         elif (mode == self.Restore):
             utils.log(utils.getString(30023) + " - " + utils.getString(30017))
 
+            #catch for if the restore point is actually a zip file
+            if(self.restore_point.split('.')[-1] == 'zip'):
+                self.progressBar.updateProgress(0, "Copying Zip Archive")
+                utils.log("copying zip file: " + self.restore_point)
+                
+                #set root to data dir home 
+                self.xbmc_vfs.set_root(xbmc.translatePath("special://temp/"))
+                
+                if(not self.xbmc_vfs.exists(xbmc.translatePath("special://temp/" + self.restore_point))):
+                    #copy just this file from the remote vfs
+                    zipFile = []
+                    zipFile.append(self.remote_base_path + self.restore_point)
+               
+                    self.backupFiles(zipFile,self.remote_vfs, self.xbmc_vfs)
+                else:
+                    utils.log("zip file exists already")
+                
+                #extract the zip file
+                zip_vfs = ZipFileSystem(xbmc.translatePath("special://temp/"+ self.restore_point),'r')
+                zip_vfs.extract(xbmc.translatePath("special://temp/"))
+                zip_vfs.cleanup()
+                
+                #set the new remote vfs and fix xbmc path
+                self.remote_vfs = XBMCFileSystem(xbmc.translatePath("special://temp/" + self.restore_point.split(".")[0] + "/"))
+                self.xbmc_vfs.set_root(xbmc.translatePath("special://home"))
+                
+            
             #for restores remote path must exist
             if(not self.remote_vfs.exists(self.remote_vfs.root_path)):
                 xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30045),self.remote_vfs.root_path)
@@ -248,10 +320,10 @@ class XbmcBackup:
                         
                         return
                 
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/keymaps'))
+                fileManager.addFile('-userdata/keymaps')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/keymaps")
                 
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/peripheral_data'))
+                fileManager.addFile('-userdata/peripheral_data')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/peripheral_data")
             
                 #this part is an oddity
@@ -261,29 +333,29 @@ class XbmcBackup:
                         fileManager.addFile(self.remote_vfs.root_path + "userdata/" + aFile)
 
             if(utils.getSetting('backup_addons') == 'true'):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/addons'))
+                fileManager.addFile('-addons')
                 fileManager.walkTree(self.remote_vfs.root_path + "addons")
 
             self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata'))
 
             if(utils.getSetting('backup_addon_data') == 'true'):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/addon_data'))
+                fileManager.addFile('-userdata/addon_data')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/addon_data")
 
             if(utils.getSetting('backup_database') == 'true'):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/Database'))
+                fileManager.addFile('-userdata/Database')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/Database")
         
             if(utils.getSetting("backup_playlists") == 'true'):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/playlists'))
+                fileManager.addFile('-userdata/playlists')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/playlists")
 
             if(utils.getSetting('backup_profiles') == 'true'):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/profiles'))
+                fileManager.addFile('-userdata/profiles')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/profiles")
                 
             if(utils.getSetting("backup_thumbnails") == "true"):
-                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/Thumbnails'))
+                fileManager.addFile('-userdata/Thumbnails')
                 fileManager.walkTree(self.remote_vfs.root_path + "userdata/Thumbnails")
 
             #add to array
@@ -321,9 +393,16 @@ class XbmcBackup:
                 self.xbmc_vfs.set_root(fileGroup['dest'])
                 self.backupFiles(fileGroup['files'],self.remote_vfs,self.xbmc_vfs)
 
+            if(self.restore_point.split('.')[-1] == 'zip'):
+                #delete the zip file and the extracted directory
+                self.xbmc_vfs.rmfile(xbmc.translatePath("special://temp/" + self.restore_point))
+                self.xbmc_vfs.rmdir(self.remote_vfs.root_path)
+
             #call update addons to refresh everything
             xbmc.executebuiltin('UpdateLocalAddons')
 
+        self.xbmc_vfs.cleanup()
+        self.remote_vfs.cleanup()
         self.progressBar.close()
 
         #reset the window setting
@@ -382,7 +461,13 @@ class XbmcBackup:
                 while(remove_num < (len(dirs) - total_backups) and not self.progressBar.checkCancel()):
                     self._updateProgress(utils.getString(30054) + " " + dirs[remove_num][1])
                     utils.log("Removing backup " + dirs[remove_num][0])
+                    
+                    if(dirs[remove_num][0].split('.')[-1] == 'zip'):
+                        #this is a file, remove it that way
+                        self.remote_vfs.rmfile(self.remote_base_path + dirs[remove_num][0])
+                    else:
                         self.remote_vfs.rmdir(self.remote_base_path + dirs[remove_num][0] + "/")
+                        
                     remove_num = remove_num + 1
 
     def _createValidationFile(self):
@@ -439,6 +524,7 @@ class FileManager:
             for aDir in dirs:
                 dirPath = xbmc.translatePath(directory + "/" + aDir)
                 file_ext = aDir.split('.')[-1]
+              
                 self.addFile("-" + dirPath)
 
                 #catch for "non directory" type files
@@ -453,7 +539,6 @@ class FileManager:
             
             #copy all the files
             for aFile in files:
-                utils.log(aFile)
                 filePath = xbmc.translatePath(directory + "/" + aFile)
                 self.addFile(filePath)
                     

resources/lib/vfs.py

@@ -2,7 +2,8 @@ import utils as utils
 import xbmc
 import xbmcvfs
 import xbmcgui
-import sys
+import zipfile
+import zlib
 from dropbox import client, rest, session
 
 APP_KEY = utils.getSetting('dropbox_key')
@@ -43,9 +44,18 @@ class Vfs:
     def rmdir(self,directory):
         return True
 
+    def rmfile(self,aFile):
+        return True
+
     def exists(self,aFile):
         return True
     
+    def rename(self,aFile,newName):
+        return True
+    
+    def cleanup(self):
+        return True
+        
 class XBMCFileSystem(Vfs):
 
     def listdir(self,directory):
@@ -60,9 +70,46 @@ class XBMCFileSystem(Vfs):
     def rmdir(self,directory):
         return xbmcvfs.rmdir(directory,True)
 
+    def rmfile(self,aFile):
+        return xbmcvfs.delete(aFile)
+
+    def rename(self,aFile,newName):
+        return xbmcvfs.rename(aFile, newName)
+
     def exists(self,aFile):
         return xbmcvfs.exists(aFile)
 
+class ZipFileSystem(Vfs):
+    zip = None
+    
+    def __init__(self,rootString,mode):
+        self.root_path = ""
+        self.zip = zipfile.ZipFile(rootString,mode=mode)
+        
+    def listdir(self,directory):
+        return [[],[]]
+    
+    def mkdir(self,directory):
+        #self.zip.write(directory[len(self.root_path):])
+        return False
+    
+    def put(self,source,dest):
+        self.zip.write(source,dest,compress_type=zipfile.ZIP_DEFLATED)
+        return True
+    
+    def rmdir(self,directory):
+        return False
+    
+    def exists(self,aFile):
+        return False
+    
+    def cleanup(self):
+        self.zip.close()
+        
+    def extract(self,path):
+        #extract zip file to path
+        self.zip.extractall(path)
+
 class DropboxFileSystem(Vfs):
     client = None
     
@@ -139,6 +186,16 @@ class DropboxFileSystem(Vfs):
 
             #finally remove the root directory
             self.client.file_delete(directory)
+            
+            return True
+        else:
+            return False
+
+    def rmFile(self,aFile):
+        aFile = self._fix_slashes(aFile)
+        if(self.client != None and self.exists(aFile)):
+            self.client.file_delete(aFile)
+            return True
         else:
             return False
 

resources/settings.xml

@@ -6,6 +6,7 @@
 		<setting id="remote_path" type="folder" label="30020" visible="eq(-2,0)" />
 		<setting id="dropbox_key" type="text" label="30028" visible="eq(-3,2)" default="" />
 		<setting id="dropbox_secret" type="text" label="30029" visible="eq(-4,2)" default="" />
+		<setting id="compress_backups" type="bool" label="Zip Archives" default="false" />
 		<setting id="backup_rotation" type="number" label="30026" default="0" />
 		<setting id="progress_mode" type="enum" label="30022" lvalues="30082|30083|30084" default="0" />
 	</category>