Merge pull request #28 from robweber/testing

Merge custom directory code

README.txt

@@ -11,6 +11,8 @@ In the addon settings you can define a remote path for the destination of your x
 
 On the Backup Selection page you can select which items from your user profile folder will be sent to the backup location. By default all are turned on except the Addon Data directory. 
 
+You can also define non-XBMC directories on your device. See "Custom Directories" for more information on how these are handled. 
+
 Scheduling: 
 
 You can also schedule backups to be completed on a set interval via the scheduling area. When it is time for the backup to run it will be executed in the background. 
@@ -21,6 +23,10 @@ Running the Program:
 
 Running the program will allow you to select Backup or Restore as a running mode. Selecting Backup will push files to your remote store using the addon settings you defined. Selecting Restore will give you a list of restore points currently in your remote destination. Selecting one will pull the files matching your selection criteria from the restore point to your local XBMC folders. 
 
+Custom Directories: 
+
+You can define custom directories that are not a part of your XBMC folder structure for backup. These create a custom_1_hash folder in your backup destination. The hash for these folders is very important. During a restore if the hash of the file path in Custom 1 does not match the hash in the restore folder it will not move the files. This is to prevent files from being restored to the wrong location in the event you change file paths in the addon settings. A dialog box will let you know if file paths do not match up. 
+
 
 Using Dropbox:
 

addon.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <addon id="script.xbmcbackup"
-    name="XBMC Backup" version="0.3.4" provider-name="robweber">
+    name="XBMC Backup" version="0.3.5" provider-name="robweber">
   <requires>
     <import addon="xbmc.python" version="2.1.0"/>
   </requires>

changelog.txt

@@ -1,3 +1,7 @@
+Version 0.3.5
+
+test of custom directories - only 1 at the moment
+
 Version 0.3.4
 
 added ability to take parameters via RunScript() or JSONRPC.Addons.ExecuteAddon()

resources/lib/backup.py

@@ -6,92 +6,13 @@ import os.path
 import time
 from vfs import XBMCFileSystem,DropboxFileSystem
 
-class FileManager:
-    fileArray = None
-    verbose_log = False
-    not_dir = ['.zip','.xsp','.rar']
-    vfs = None
-    
-    def __init__(self,vfs):
-        self.vfs = vfs
-        
-    def createFileList(self):
-        self.fileArray = []
-        self.verbose_log = utils.getSetting("verbose_log") == 'true'
-       
-        #figure out which syncing options to run
-        if(utils.getSetting('backup_addons') == 'true'):
-            self.addFile("-addons")
-            self.walkTree(self.vfs.root_path + "addons/")
-
-        self.addFile("-userdata")
-        
-        if(utils.getSetting('backup_addon_data') == 'true'):
-            self.addFile("-userdata/addon_data")
-            self.walkTree(self.vfs.root_path + "userdata/addon_data/")
-           
-        if(utils.getSetting('backup_database') == 'true'):
-	    self.addFile("-userdata/Database")
-            self.walkTree(self.vfs.root_path + "userdata/Database")
-        
-        if(utils.getSetting("backup_playlists") == 'true'):
-	    self.addFile("-userdata/playlists")
-	    self.walkTree(self.vfs.root_path + "userdata/playlists")
-			
-        if(utils.getSetting("backup_thumbnails") == "true"):
-	    self.addFile("-userdata/Thumbnails")
-	    self.walkTree(self.vfs.root_path + "userdata/Thumbnails")
-		
-        if(utils.getSetting("backup_config") == "true"):
-            self.addFile("-userdata/keymaps")
-            self.walkTree(self.vfs.root_path + "userdata/keymaps")
-
-            self.addFile("-userdata/peripheral_data")
-            self.walkTree(self.vfs.root_path + "userdata/peripheral_data")
-            
-	    #this part is an oddity
-            dirs,configFiles = self.vfs.listdir(self.vfs.root_path + "userdata/")
-	    for aFile in configFiles:
-		if(aFile.endswith(".xml")):
-		    self.addFile("userdata/" + aFile)
-        
-    def walkTree(self,directory):
-        dirs,files = self.vfs.listdir(directory)
-        
-        #create all the subdirs first
-        for aDir in dirs:
-            dirPath = xbmc.translatePath(directory + "/" + aDir)
-            file_ext = aDir.split('.')[-1]
-            self.addFile("-" + dirPath[len(self.vfs.root_path):])  
-            #catch for "non directory" type files
-            if (not any(file_ext in s for s in self.not_dir)):
-                self.walkTree(dirPath)  
-            
-        #copy all the files
-        for aFile in files:
-            filePath = xbmc.translatePath(directory + "/" + aFile)
-            self.addFile(filePath[len(self.vfs.root_path):])
-                    
-    def addFile(self,filename):
-        try:
-            filename = filename.decode('UTF-8')
-        except UnicodeDecodeError:
-            filename = filename.decode('ISO-8859-2')
-            
-        #write the full remote path name of this file
-        utils.log("Add File: " + filename,xbmc.LOGDEBUG)
-        self.fileArray.append(filename)
-
-    def getFileList(self):
-       return self.fileArray
-
 class XbmcBackup:
     #constants for initiating a back or restore
     Backup = 0
     Restore = 1
 
     #remote file system
-    local_vfs = None
+    xbmc_vfs = None
     remote_vfs = None
     restoreFile = None
     
@@ -102,26 +23,21 @@ class XbmcBackup:
 
     fileManager = None
     restore_point = None
-    
-    def __init__(self):
-        self.local_vfs = XBMCFileSystem()
-        self.local_vfs.set_root(xbmc.translatePath("special://home"))
 
-        self.configureVFS()
+    def __init__(self):
-        
+        self.xbmc_vfs = XBMCFileSystem(xbmc.translatePath('special://home'))
+
+        self.configureRemote()
         utils.log(utils.getString(30046))
 
-    def configureVFS(self):
+    def configureRemote(self):
         if(utils.getSetting('remote_selection') == '1'):
-            self.remote_vfs = XBMCFileSystem()
+            self.remote_vfs = XBMCFileSystem(utils.getSetting('remote_path_2'))
-            self.remote_vfs.set_root(utils.getSetting('remote_path_2'))
 	    utils.setSetting("remote_path","")
         elif(utils.getSetting('remote_selection') == '0'):
-            self.remote_vfs = XBMCFileSystem()
+            self.remote_vfs = XBMCFileSystem(utils.getSetting("remote_path"))
-            self.remote_vfs.set_root(utils.getSetting("remote_path"))
         elif(utils.getSetting('remote_selection') == '2'):
-            self.remote_vfs = DropboxFileSystem()
+            self.remote_vfs = DropboxFileSystem("/")
-            self.remote_vfs.set_root('/')
 
     def listBackups(self):
         result = list()
@@ -138,7 +54,6 @@ class XbmcBackup:
         self.restore_point = restore_point
 
     def run(self,mode=-1,runSilent=False):
-
         #append backup folder name
         remote_base_path = ""
         progressBarTitle = utils.getString(30010) + " - "
@@ -154,7 +69,7 @@ class XbmcBackup:
 	    self.remote_vfs = None
 	    return
 
-        utils.log(utils.getString(30047) + ": " + self.local_vfs.root_path)
+        utils.log(utils.getString(30047) + ": " + self.xbmc_vfs.root_path)
         utils.log(utils.getString(30048) + ": " + self.remote_vfs.root_path)
 
         #check if we should use the progress bar
@@ -162,17 +77,78 @@ class XbmcBackup:
             self.progressBar = xbmcgui.DialogProgress()
             self.progressBar.create(progressBarTitle,utils.getString(30049) + "......")
 
-        #run the correct mode
         if(mode == self.Backup):
             utils.log(utils.getString(30023) + " - " + utils.getString(30016))
-            self.fileManager = FileManager(self.local_vfs)
+            #check if remote path exists
-
-            #for backups check if remote path exists
             if(self.remote_vfs.exists(self.remote_vfs.root_path)):
-                #this will fail - need a disclaimer here
+                #may be data in here already
                 utils.log(utils.getString(30050))
+            else:
+                #make the remote directory
+                self.remote_vfs.mkdir(self.remote_vfs.root_path)
 
-            self.syncFiles()
+            utils.log(utils.getString(30051))
+            allFiles = []
+            fileManager = FileManager(self.xbmc_vfs)
+         
+            #go through each of the user selected items and write them to the backup store
+            if(utils.getSetting('backup_addons') == 'true'):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "addons")
+                fileManager.walkTree(xbmc.translatePath('special://home/addons'))
+
+            self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata")
+
+            if(utils.getSetting('backup_addon_data') == 'true'):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/addon_data")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/addon_data'))
+
+            if(utils.getSetting('backup_database') == 'true'):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/Database")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/Database'))
+        
+            if(utils.getSetting("backup_playlists") == 'true'):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/playlists")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/playlists'))
+                
+            if(utils.getSetting("backup_thumbnails") == "true"):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/Thumbnails")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/Thumbnails'))
+	  
+            if(utils.getSetting("backup_config") == "true"):
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/keymaps")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/keymaps'))
+                
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "userdata/peripheral_data")
+                fileManager.walkTree(xbmc.translatePath('special://home/userdata/peripheral_data'))
+            
+                #this part is an oddity
+                dirs,configFiles = self.xbmc_vfs.listdir(xbmc.translatePath('special://home/userdata/'))
+                for aFile in configFiles:
+                    if(aFile.endswith(".xml")):
+                        fileManager.addFile(xbmc.translatePath('special://home/userdata/') + aFile)
+
+            #add to array
+            self.filesTotal = fileManager.size()
+            allFiles.append({"source":self.xbmc_vfs.root_path,"dest":self.remote_vfs.root_path,"files":fileManager.getFiles()})
+
+            #check if there are custom directories
+            if(utils.getSetting('backup_custom_dir') != ''):
+
+                #create a special remote path with hash                
+                self.xbmc_vfs.set_root(utils.getSetting('backup_custom_dir'))
+                self.remote_vfs.mkdir(self.remote_vfs.root_path + "custom_1_" + self._createCRC(self.xbmc_vfs.root_path))
+                self.remote_vfs.set_root(self.remote_vfs.root_path + "custom_1_" + self._createCRC(self.xbmc_vfs.root_path))
+
+                fileManager.walkTree(self.xbmc_vfs.root_path)
+                self.filesTotal = self.filesTotal + fileManager.size()
+                allFiles.append({"source":self.xbmc_vfs.root_path,"dest":self.remote_vfs.root_path,"files":fileManager.getFiles()})
+
+            #backup all the files
+            self.filesLeft = self.filesTotal
+            for fileGroup in allFiles:
+                self.xbmc_vfs.set_root(fileGroup['source'])
+                self.remote_vfs.set_root(fileGroup['dest'])
+                self.backupFiles(fileGroup['files'],self.xbmc_vfs,self.remote_vfs)
 
             #remove old backups
             total_backups = int(utils.getSetting('backup_rotation'))
@@ -186,82 +162,178 @@ class XbmcBackup:
                     self.filesTotal = self.filesTotal + remove_num + 1
 
                     #update the progress bar if it is available
-                    while(remove_num >= 0 and not self.checkCancel()):
+                    while(remove_num >= 0 and not self._checkCancel()):
-                        self.updateProgress(utils.getString(30054) + " " + dirs[remove_num])
+                        self._updateProgress(utils.getString(30054) + " " + dirs[remove_num])
                         utils.log("Removing backup " + dirs[remove_num])
                         self.remote_vfs.rmdir(remote_base_path + dirs[remove_num] + "/")
                         remove_num = remove_num - 1
-                        
+
-                
+        elif (mode == self.Restore):
-        else:
             utils.log(utils.getString(30023) + " - " + utils.getString(30017))
-            self.fileManager = FileManager(self.remote_vfs)
 
             #for restores remote path must exist
-            if(self.remote_vfs.exists(self.remote_vfs.root_path)):
+            if(not self.remote_vfs.exists(self.remote_vfs.root_path)):
-                self.restoreFiles()
-            else:
                 xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30045),self.remote_vfs.root_path)
+                return
+
+            utils.log(utils.getString(30051))
+            allFiles = []
+            fileManager = FileManager(self.remote_vfs)
+         
+            #go through each of the user selected items and write them to the backup store
+            if(utils.getSetting('backup_addons') == 'true'):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/addons'))
+                fileManager.walkTree(self.remote_vfs.root_path + "addons")
+
+            self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata'))
+
+            if(utils.getSetting('backup_addon_data') == 'true'):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/addon_data'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/addon_data")
+
+            if(utils.getSetting('backup_database') == 'true'):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/Database'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/Database")
+        
+            if(utils.getSetting("backup_playlists") == 'true'):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/playlists'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/playlists")
+                
+            if(utils.getSetting("backup_thumbnails") == "true"):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/Thumbnails'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/Thumbnails")
+	  
+            if(utils.getSetting("backup_config") == "true"):
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/keymaps'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/keymaps")
+                
+                self.xbmc_vfs.mkdir(xbmc.translatePath('special://home/userdata/peripheral_data'))
+                fileManager.walkTree(self.remote_vfs.root_path + "userdata/peripheral_data")
+            
+                #this part is an oddity
+                dirs,configFiles = self.remote_vfs.listdir(self.remote_vfs.root_path + "userdata/")
+                for aFile in configFiles:
+                    if(aFile.endswith(".xml")):
+                        fileManager.addFile(self.remote_vfs.root_path + "userdata/" + aFile)
+
+            #add to array
+            self.filesTotal = fileManager.size()
+            allFiles.append({"source":self.remote_vfs.root_path,"dest":self.xbmc_vfs.root_path,"files":fileManager.getFiles()})    
+
+            #check if there are custom directories
+            if(utils.getSetting('backup_custom_dir') != ''):
+
+                self.xbmc_vfs.set_root(utils.getSetting('backup_custom_dir'))
+                if(self.remote_vfs.exists(self.remote_vfs.root_path + "custom_1_" + self._createCRC(self.xbmc_vfs.root_path))):
+                    #index files to restore
+                    self.remote_vfs.set_root(self.remote_vfs.root_path + "custom_1_" + self._createCRC(self.xbmc_vfs.root_path))
+
+                    fileManager.walkTree(self.remote_vfs.root_path)
+                    self.filesTotal = self.filesTotal + fileManager.size()
+                    allFiles.append({"source":self.remote_vfs.root_path,"dest":self.xbmc_vfs.root_path,"files":fileManager.getFiles()})
+                else:
+                    xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30045),self.remote_vfs.root_path + "custom_1_" + self._createCRC(utils.getSetting('backup_custom_dir')))
+
+            #restore all the files
+            self.filesLeft = self.filesTotal
+            for fileGroup in allFiles:
+                self.remote_vfs.set_root(fileGroup['source'])
+                self.xbmc_vfs.set_root(fileGroup['dest'])
+                self.backupFiles(fileGroup['files'],self.remote_vfs,self.xbmc_vfs)
+
+            #call update addons to refresh everything
+            xbmc.executebuiltin('UpdateLocalAddons')
 
         if(utils.getSetting('run_silent') == 'false' and not runSilent):
             self.progressBar.close()
-        
-    def syncFiles(self):
-        
-        #make the remote directory
-        self.remote_vfs.mkdir(self.remote_vfs.root_path)
 
-        utils.log(utils.getString(30051))
+    def backupFiles(self,fileList,source,dest):
-        self.fileManager.createFileList()
-
-        allFiles = self.fileManager.getFileList()
-
-        #write list from local to remote
-        self.writeFiles(allFiles,self.local_vfs,self.remote_vfs)
-        
-    def restoreFiles(self):
-        self.fileManager.createFileList()
-
-        utils.log(utils.getString(30051))
-        allFiles = self.fileManager.getFileList()
-
-        #write list from remote to local
-        self.writeFiles(allFiles,self.remote_vfs,self.local_vfs)
-
-        #call update addons to refresh everything
-        xbmc.executebuiltin('UpdateLocalAddons')
-        
-    def writeFiles(self,fileList,source,dest):
         utils.log("Writing files to: " + dest.root_path)
-        self.filesTotal = len(fileList)
+        utils.log("Source: " + source.root_path)
-        self.filesLeft = self.filesTotal
-
-        #write each file from source to destination
         for aFile in fileList:
-            if(not self.checkCancel()):
+            if(not self._checkCancel()):
-                utils.log('Writing file: ' + source.root_path + aFile,xbmc.LOGDEBUG)
+                utils.log('Writing file: ' + aFile,xbmc.LOGDEBUG)
-                self.updateProgress(aFile)
+                self._updateProgress(aFile)
-                if (aFile.startswith("-")):
+                if(aFile.startswith("-")):
-                    dest.mkdir(dest.root_path + aFile[1:])
+                    dest.mkdir(dest.root_path + aFile[len(source.root_path) + 1:])
                 else:
                     if(isinstance(source,DropboxFileSystem)):
                         #if copying from dropbox we need the file handle, use get_file
-                        source.get_file(source.root_path + aFile,dest.root_path + aFile)
+                        source.get_file(aFile,dest.root_path + aFile[len(source.root_path):])
                     else:
                         #copy using normal method
-                        dest.put(source.root_path + aFile,dest.root_path + aFile)
+                        dest.put(aFile,dest.root_path + aFile[len(source.root_path):])
 
-    def updateProgress(self,message=''):
+    def _createCRC(self,string):
+        #create hash from string
+        string = string.lower()        
+        bytes = bytearray(string.encode())
+        crc = 0xffffffff;
+        for b in bytes:
+            crc = crc ^ (b << 24)          
+            for i in range(8):
+                if (crc & 0x80000000 ):                 
+                    crc = (crc << 1) ^ 0x04C11DB7                
+                else:
+                    crc = crc << 1;                        
+                    crc = crc & 0xFFFFFFFF
+        
+        return '%08x' % crc
+    
+    def _updateProgress(self,message=''):
         self.filesLeft = self.filesLeft - 1
 
         #update the progress bar
         if(self.progressBar != None):
             self.progressBar.update(int((float(self.filesTotal - self.filesLeft)/float(self.filesTotal)) * 100),message)
             
-    def checkCancel(self):
+    def _checkCancel(self):
         result = False
 
         if(self.progressBar != None):
             result = self.progressBar.iscanceled()
 
         return result
+
+class FileManager:
+    fileArray = []
+    not_dir = ['.zip','.xsp','.rar']
+    vfs = None
+
+    def __init__(self,vfs):
+        self.vfs = vfs
+
+    def walkTree(self,directory):
+        dirs,files = self.vfs.listdir(directory)
+        
+        #create all the subdirs first
+        for aDir in dirs:
+            dirPath = xbmc.translatePath(directory + "/" + aDir)
+            file_ext = aDir.split('.')[-1]
+            self.addFile("-" + dirPath)  
+            #catch for "non directory" type files
+            if (not any(file_ext in s for s in self.not_dir)):
+                self.walkTree(dirPath)  
+            
+        #copy all the files
+        for aFile in files:
+            filePath = xbmc.translatePath(directory + "/" + aFile)
+            self.addFile(filePath)
+                    
+    def addFile(self,filename):
+        try:
+            filename = filename.decode('UTF-8')
+        except UnicodeDecodeError:
+            filename = filename.decode('ISO-8859-2')
+            
+        #write the full remote path name of this file
+        utils.log("Add File: " + filename,xbmc.LOGDEBUG)
+        self.fileArray.append(filename)
+
+    def getFiles(self):
+        result = self.fileArray
+        self.fileArray = []
+        return result
+
+    def size(self):
+        return len(self.fileArray)

resources/lib/vfs.py

@@ -11,6 +11,9 @@ APP_SECRET = utils.getSetting('dropbox_secret')
 class Vfs:
     root_path = None
 
+    def __init__(self,rootString):
+        self.set_root(rootString)
+        
     def set_root(self,rootString):
         old_root = self.root_path
         self.root_path = rootString
@@ -44,7 +47,7 @@ class Vfs:
         return True
         
 class XBMCFileSystem(Vfs):
-    
+
     def listdir(self,directory):
         return xbmcvfs.listdir(directory)
 
@@ -63,7 +66,11 @@ class XBMCFileSystem(Vfs):
 class DropboxFileSystem(Vfs):
     client = None
     
-    def __init__(self):
+    def __init__(self,rootString):
+        self.set_root(rootString)
+        self.setup()
+
+    def setup(self):
         if(APP_KEY == '' or APP_SECRET == ''):
             xbmcgui.Dialog().ok(utils.getString(30010),utils.getString(30058),utils.getString(30059))
             return
@@ -94,7 +101,6 @@ class DropboxFileSystem(Vfs):
         except:
             #this didn't work, delete the token file
             self.deleteToken()
-            
 
     def listdir(self,directory):
         if(self.client != None and self.exists(directory)):
@@ -114,6 +120,7 @@ class DropboxFileSystem(Vfs):
             
 
     def mkdir(self,directory):
+        directory = self._fix_slashes(directory)
         if(self.client != None):
             if(not self.exists(directory)):
                 self.client.file_create_folder(directory)
@@ -122,12 +129,14 @@ class DropboxFileSystem(Vfs):
             return False
 
     def rmdir(self,directory):
+        directory = self._fix_slashes(directory)
         if(self.client != None and self.exists(directory)):
             self.client.file_delete(directory)
         else:
             return False
 
     def exists(self,aFile):
+        aFile = self._fix_slashes(aFile)
         if(self.client != None):
             try:
                 meta_data = self.client.metadata(aFile)
@@ -138,7 +147,9 @@ class DropboxFileSystem(Vfs):
         else:
             return False
 
-    def put(self,source,dest):        
+    def put(self,source,dest):
+        dest = self._fix_slashes(dest)
+        
         if(self.client != None):
             f = open(source,'rb')
             try:
@@ -158,9 +169,13 @@ class DropboxFileSystem(Vfs):
             f = self.client.get_file(source).read()
             out.write(f)
             out.close()
+            return True
         else:
             return False
-        
+
+    def _fix_slashes(self,filename):
+        return filename.replace('\\','/')
+    
     def setToken(self,key,secret):
         #write the token files
         token_file = open(xbmc.translatePath(utils.data_dir() + "tokens.txt"),'w')

resources/settings.xml

@@ -16,6 +16,7 @@
 		<setting id="backup_playlists" type="bool" label="30033" default="true" />
 		<setting id="backup_thumbnails" type="bool" label="30034" default="true" />
 		<setting id="backup_config" type="bool" label="30035" default="true" />
+		<setting id="backup_custom_dir" type="folder" label="Additional Directory" default="" />
 	</category>
 	<category id="scheduling" label="30013">
 		<setting id="enable_scheduler" type="bool" label="30060" default="false" />