From 815b15840c4dfe60ba4c6c85ad2fab7b1b702309 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 30 Sep 2015 10:51:22 +0200 Subject: [PATCH] Version 1.0.0 --- CorrectPath.go | 9 + Main.go | 34 ++- ReadFlags.go | 3 +- Sync/ComparePath.go | 11 - Sync/{Connect.go => ConnectSSH.go} | 0 Sync/NormalisePath.go | 12 ++ Sync/ReadYesNoAnswer.go | 23 ++ Sync/Synchronise.go | 334 ++++++++++++++--------------- Sync/WalkerLocal.go | 10 + Variables.go | 1 + 10 files changed, 246 insertions(+), 191 deletions(-) create mode 100644 CorrectPath.go rename Sync/{Connect.go => ConnectSSH.go} (100%) create mode 100644 Sync/NormalisePath.go create mode 100644 Sync/ReadYesNoAnswer.go create mode 100644 Sync/WalkerLocal.go diff --git a/CorrectPath.go b/CorrectPath.go new file mode 100644 index 0000000..856ef1b --- /dev/null +++ b/CorrectPath.go @@ -0,0 +1,9 @@ +package main + +func correctPath(path string) string { + if path[len(path)-1:] == `/` || path[len(path)-1:] == `\` { + return path[:len(path)-1] + } else { + return path + } +} diff --git a/Main.go b/Main.go index 6b90c48..c8b9389 100644 --- a/Main.go +++ b/Main.go @@ -12,7 +12,7 @@ import ( func main() { // Show the current version: - fmt.Println(`Sync v1.0.0`) + log.Println(`Sync v1.0.0`) // Allow Go to use all CPUs: runtime.GOMAXPROCS(runtime.NumCPU()) @@ -26,6 +26,21 @@ func main() { return } + // Should I use the current working dir? + if localDir == `.` { + if currentWD, currentWDError := os.Getwd(); currentWDError != nil { + log.Println("Cannot use the current working directory as local directory: " + currentWDError.Error()) + return + } else { + log.Println("I use the current working directory as local directory: " + currentWD) + localDir = currentWD + } + } + + // Remove trailing separators from both directories + localDir = correctPath(localDir) + remoteDir = correctPath(remoteDir) + // Check if local dir exist if dirInfo, dirError := os.Stat(localDir); dirError != nil { log.Println("There is an error with the local directory: " + dirError.Error()) @@ -41,13 +56,26 @@ func main() { for true { if password == `` { // Promt for the password: - fmt.Println(`Please provide the password for the connection:`) + fmt.Print(`Please provide the password for the connection: `) fmt.Scanln(&password) } else { break } } + // Give some information about the state + if supervised { + log.Println("I use the supervised mode.") + } else { + log.Println("I do not use the supervised mode.") + } + + if pushOnly { + log.Println("I use the push only mode i.e. backup mode. Any remote change will be ignored.") + } else { + log.Println("I use the full mode and consider also remote changes.") + } + // Create the SSH configuration: Sync.SetPassword4Callback(password) config := &ssh.ClientConfig{ @@ -67,6 +95,6 @@ func main() { } defer ssh.Close() - Sync.Synchronise(ssh, supervised, localDir, remoteDir) + Sync.Synchronise(ssh, supervised, pushOnly, localDir, remoteDir) log.Println("Synchronising done.") } diff --git a/ReadFlags.go b/ReadFlags.go index 5b9365e..6904791 100644 --- a/ReadFlags.go +++ b/ReadFlags.go @@ -8,8 +8,9 @@ func readFlags() { flag.StringVar(&serverAddrString, `server`, `127.0.0.1:22`, `The (remote) SSH server, e.g. 'my.host.com', 'my.host.com:22', '127.0.0.1:22', 'localhost:22'.`) flag.StringVar(&username, `user`, `username`, `The user's name for the SSH server.`) flag.StringVar(&password, `pwd`, ``, `The user's password for the SSH server. You can omit these argument. Thus, the program asks for the password on demand.`) - flag.StringVar(&localDir, `localDir`, ``, `The local directory which should be synced.`) + flag.StringVar(&localDir, `localDir`, `.`, `The local directory which should be synced. Use . for the current working directory.`) flag.StringVar(&remoteDir, `remoteDir`, ``, `The remote directory which should be synced.`) flag.BoolVar(&supervised, `supervised`, true, `Use the supervised mode? The algorithm asks you before any change.`) + flag.BoolVar(&pushOnly, `pushOnly`, true, `Use the push only mode, i.e. backup mode. Ignore any change on the remote side!`) flag.Parse() } diff --git a/Sync/ComparePath.go b/Sync/ComparePath.go index e82c27f..babc0f7 100644 --- a/Sync/ComparePath.go +++ b/Sync/ComparePath.go @@ -1,18 +1,7 @@ package Sync -import ( - "path/filepath" - "strings" -) - func comparePath(localBase, localPath, remoteBase, remotePath string) bool { localCompare := normalisePath(localBase, localPath) remoteCompare := normalisePath(remoteBase, remotePath) return localCompare == remoteCompare } - -func normalisePath(base, path string) string { - result := strings.Replace(path, base, ``, 1) - result = filepath.ToSlash(result) - return strings.ToLower(result) -} diff --git a/Sync/Connect.go b/Sync/ConnectSSH.go similarity index 100% rename from Sync/Connect.go rename to Sync/ConnectSSH.go diff --git a/Sync/NormalisePath.go b/Sync/NormalisePath.go new file mode 100644 index 0000000..6c21541 --- /dev/null +++ b/Sync/NormalisePath.go @@ -0,0 +1,12 @@ +package Sync + +import ( + "path/filepath" + "strings" +) + +func normalisePath(base, path string) string { + result := strings.Replace(path, base, ``, 1) + result = filepath.ToSlash(result) + return strings.ToLower(result) +} diff --git a/Sync/ReadYesNoAnswer.go b/Sync/ReadYesNoAnswer.go new file mode 100644 index 0000000..d4f3279 --- /dev/null +++ b/Sync/ReadYesNoAnswer.go @@ -0,0 +1,23 @@ +package Sync + +import ( + "fmt" + "strings" +) + +func readYesNoAnswer(defaultAnswer bool) bool { // true := yes + answer := `` + if _, scanError := fmt.Scan(&answer); scanError != nil { + return defaultAnswer + } + + if answer == `` || answer == ` ` { + return defaultAnswer + } + + if strings.ToLower(answer) == `y` { + return true + } + + return false +} diff --git a/Sync/Synchronise.go b/Sync/Synchronise.go index 4a1133a..f6674a6 100644 --- a/Sync/Synchronise.go +++ b/Sync/Synchronise.go @@ -13,11 +13,7 @@ import ( "time" ) -var ( - test string = "" -) - -func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { +func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) { /* Algorithm: - Get all local files and dirs @@ -65,11 +61,8 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { log.Printf("Found %d local files.\n", len(localFiles)) // - // Read the remote files ============================================================================================ + // Connect to the server ============================================================================================ // - log.Println("Try to read all remote files now...") - remoteFiles = make(map[string]os.FileInfo) - sftp, sftpError := sftp.NewClient(ssh) if sftpError != nil { log.Println("Was not able to connect to the server: " + sftpError.Error()) @@ -78,6 +71,11 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { defer sftp.Close() + // + // Read the remote files ============================================================================================ + // + log.Println("Try to read all remote files now...") + remoteFiles = make(map[string]os.FileInfo) counterRemoteFile := 0 walker := sftp.Walk(remoteDir) for walker.Step() { @@ -213,66 +211,68 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { // // Free local space ============================================================================================ // - deleteLocalFiles := make([]string, 0) - for localFileNormalised, localFileInfo := range localFilesNormalised { - remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] - if remoteFileNormaliesed == nil { - if localFileInfo.ModTime().UTC().Before(ndr) { - deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) + if !pushOnly { + deleteLocalFiles := make([]string, 0) + for localFileNormalised, localFileInfo := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed == nil { + if localFileInfo.ModTime().UTC().Before(ndr) { + deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) + } } } - } - log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) + log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) - if len(deleteLocalFiles) > 0 { - sort.Strings(deleteLocalFiles) - shouldDeleteLocalFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range deleteLocalFiles { - fmt.Println(normalised2localFiles[file]) + if len(deleteLocalFiles) > 0 { + sort.Strings(deleteLocalFiles) + shouldDeleteLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range deleteLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I delete these local files? ") + shouldDeleteLocalFiles = readYesNoAnswer(false) } - fmt.Print("Should I delete these local files? ") - shouldDeleteLocalFiles = readYesNoAnswer(false) - } + if shouldDeleteLocalFiles { + for _, localFileNormalised := range deleteLocalFiles { - if shouldDeleteLocalFiles { - for _, localFileNormalised := range deleteLocalFiles { + // Skip all directories: + if localFilesNormalised[localFileNormalised].IsDir() { + continue + } - // Skip all directories: - if localFilesNormalised[localFileNormalised].IsDir() { - continue + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised]) + } } - removeError := os.Remove(normalised2localFiles[localFileNormalised]) - if removeError != nil { - log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised]) - } - } + for _, localFileNormalised := range deleteLocalFiles { - for _, localFileNormalised := range deleteLocalFiles { + // Skip all files: + if !localFilesNormalised[localFileNormalised].IsDir() { + continue + } - // Skip all files: - if !localFilesNormalised[localFileNormalised].IsDir() { - continue + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised]) + } } - removeError := os.Remove(normalised2localFiles[localFileNormalised]) - if removeError != nil { - log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised]) + for _, localFileNormalised := range deleteLocalFiles { + delete(localFiles, normalised2localFiles[localFileNormalised]) + delete(localFilesNormalised, localFileNormalised) } } - - for _, localFileNormalised := range deleteLocalFiles { - delete(localFiles, normalised2localFiles[localFileNormalised]) - delete(localFilesNormalised, localFileNormalised) - } } } @@ -280,81 +280,83 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { // Download new files ============================================================================================ // - downloadRemoteFiles := make([]string, 0) - for remoteFileNormalised, _ := range remoteFilesNormalised { - localFileNormaliesed := localFilesNormalised[remoteFileNormalised] - if localFileNormaliesed == nil { - downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) - } - } - - log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles)) - - if len(downloadRemoteFiles) > 0 { - sort.Strings(downloadRemoteFiles) - shouldDownloadRemoteFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range downloadRemoteFiles { - fmt.Println(normalised2remoteFiles[file]) + if !pushOnly { + downloadRemoteFiles := make([]string, 0) + for remoteFileNormalised, _ := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed == nil { + downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) } - - fmt.Print("Should I download these new remote files? ") - shouldDownloadRemoteFiles = readYesNoAnswer(false) } - if shouldDownloadRemoteFiles { + log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles)) - // 1. Create all new directories - for _, remoteFileNormalised := range downloadRemoteFiles { - - // Skip all files - if !remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue + if len(downloadRemoteFiles) > 0 { + sort.Strings(downloadRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range downloadRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) } - newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - log.Printf("Try to create the new local directory %s...\n", newLocalDir) - if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil { - log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error()) - } + fmt.Print("Should I download these new remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) } - // 2. All new files - for _, remoteFileNormalised := range downloadRemoteFiles { + if shouldDownloadRemoteFiles { - // Skip all directories - if remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue + // 1. Create all new directories + for _, remoteFileNormalised := range downloadRemoteFiles { + + // Skip all files + if !remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + log.Printf("Try to create the new local directory %s...\n", newLocalDir) + if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil { + log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error()) + } } - log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) - remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) - if remoteFileHandleError != nil { - log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) - continue - } + // 2. All new files + for _, remoteFileNormalised := range downloadRemoteFiles { - _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - newLocalFile := filepath.Join(localDir, path, filename) - localFileHandle, localFileHandleError := os.Create(newLocalFile) - if localFileHandleError != nil { - log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } + // Skip all directories + if remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } + + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + newLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(newLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } - _, copyError := io.Copy(localFileHandle, remoteFileHandle) - if copyError != nil { - log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error()) remoteFileHandle.Close() localFileHandle.Close() - continue } - - remoteFileHandle.Close() - localFileHandle.Close() } } } @@ -447,60 +449,62 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { // Changed files on the remote side ============================================================================================ // - changedRemoteFiles := make([]string, 0) - for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { - localFileNormaliesed := localFilesNormalised[remoteFileNormalised] - if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { - if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { - changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) + if !pushOnly { + changedRemoteFiles := make([]string, 0) + for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { + if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { + changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) + } } } - } - log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) + log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) - if len(changedRemoteFiles) > 0 { - sort.Strings(changedRemoteFiles) - shouldDownloadRemoteFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range changedRemoteFiles { - fmt.Println(normalised2remoteFiles[file]) + if len(changedRemoteFiles) > 0 { + sort.Strings(changedRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range changedRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I download these changed remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) } - fmt.Print("Should I download these changed remote files? ") - shouldDownloadRemoteFiles = readYesNoAnswer(false) - } + if shouldDownloadRemoteFiles { + for _, remoteFileNormalised := range changedRemoteFiles { + log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } - if shouldDownloadRemoteFiles { - for _, remoteFileNormalised := range changedRemoteFiles { - log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) - remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) - if remoteFileHandleError != nil { - log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) - continue - } + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + existingLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(existingLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } - _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - existingLocalFile := filepath.Join(localDir, path, filename) - localFileHandle, localFileHandleError := os.Create(existingLocalFile) - if localFileHandleError != nil { - log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } - _, copyError := io.Copy(localFileHandle, remoteFileHandle) - if copyError != nil { - log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error()) remoteFileHandle.Close() localFileHandle.Close() - continue } - - remoteFileHandle.Close() - localFileHandle.Close() } } } @@ -568,25 +572,3 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { } } } - -func walkerlocal(path string, info os.FileInfo, err error) error { - localFiles[path] = info - return nil -} - -func readYesNoAnswer(defaultAnswer bool) bool { // true := yes - answer := `` - if _, scanError := fmt.Scan(&answer); scanError != nil { - return defaultAnswer - } - - if answer == `` || answer == ` ` { - return defaultAnswer - } - - if strings.ToLower(answer) == `y` { - return true - } - - return false -} diff --git a/Sync/WalkerLocal.go b/Sync/WalkerLocal.go new file mode 100644 index 0000000..823e924 --- /dev/null +++ b/Sync/WalkerLocal.go @@ -0,0 +1,10 @@ +package Sync + +import ( + "os" +) + +func walkerlocal(path string, info os.FileInfo, err error) error { + localFiles[path] = info + return nil +} diff --git a/Variables.go b/Variables.go index 82592f4..80f6b82 100644 --- a/Variables.go +++ b/Variables.go @@ -7,4 +7,5 @@ var ( localDir = `` // The local directory remoteDir = `` // The remote directory supervised = true // Should the tool work supervised? + pushOnly = true // Pushes only local changes to the remote i.e. backup mode )