From 5cd29096774645764edde1423db083f39c461cd2 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 16 Mar 2016 08:03:29 +0100 Subject: [PATCH] Version 1.2.0 In case the password must provided interactively, it is now invisible. Added also exit codes in case of errors for a better usage and automation e.g. with Docker containers. Finally, improved the console output for e.g. Docker containers. --- Main.go | 16 +- Sync/Synchronise.go | 1149 ++++++++++++++++++++++--------------------- 2 files changed, 589 insertions(+), 576 deletions(-) diff --git a/Main.go b/Main.go index a17cce2..ca56bff 100644 --- a/Main.go +++ b/Main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "github.com/SommerEngineering/Sync/Sync" + "github.com/howeyc/gopass" "golang.org/x/crypto/ssh" "log" "os" @@ -12,7 +13,7 @@ import ( func main() { // Show the current version: - log.Println(`Sync v1.1.0`) + log.Println(`Sync v1.2.0`) // Allow Go to use all CPUs: runtime.GOMAXPROCS(runtime.NumCPU()) @@ -23,6 +24,7 @@ func main() { // Check if the directories are provided: if localDir == `` || remoteDir == `` { log.Println(`Please provide the local and remote directory.`) + os.Exit(1) return } @@ -30,6 +32,7 @@ func main() { if localDir == `.` { if currentWD, currentWDError := os.Getwd(); currentWDError != nil { log.Println("Cannot use the current working directory as local directory: " + currentWDError.Error()) + os.Exit(2) return } else { log.Println("I use the current working directory as local directory: " + currentWD) @@ -44,10 +47,12 @@ func main() { // Check if local dir exist if dirInfo, dirError := os.Stat(localDir); dirError != nil { log.Println("There is an error with the local directory: " + dirError.Error()) + os.Exit(3) return } else { if !dirInfo.IsDir() { log.Println("There is an error with the local directory: You provided a file instead!") + os.Exit(4) return } } @@ -57,7 +62,13 @@ func main() { if password == `` { // Promt for the password: fmt.Print(`Please provide the password for the connection: `) - fmt.Scanln(&password) + if pass, errPass := gopass.GetPasswd(); errPass != nil { + log.Println(`There was an error reading the password securely: ` + errPass.Error()) + os.Exit(5) + return + } else { + password = string(pass) + } } else { break } @@ -91,6 +102,7 @@ func main() { ssh := Sync.ConnectSSH(config, serverAddrString) if ssh == nil { log.Println(`It was not possible to connect to the SSH server.`) + os.Exit(6) return } diff --git a/Sync/Synchronise.go b/Sync/Synchronise.go index f6674a6..4f8f8c9 100644 --- a/Sync/Synchronise.go +++ b/Sync/Synchronise.go @@ -1,574 +1,575 @@ -package Sync - -import ( - "fmt" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - "io" - "log" - "os" - "path/filepath" - "sort" - "strings" - "time" -) - -func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) { - /* - Algorithm: - - Get all local files and dirs - - Get all remote files and dirs - - Determine: ndl := newest mod time of all local files - - Determine: ndr := newest mod time of all remote files - - Free remote space - - Are remote files present, which are locally not present - and these remote files are older than ndl? - - Meaning: You have worked locally and remotely are older files which are not locally present. - Interpretation of change: You have deleted these files locally, because you own local - files which are newer that these files. - - delete these remote files! (may ask the user) - - Free local space - - Are local files present, which are remotely not present - and these local files are older than ndr? - - Meaning: You have worked remotely (or have synced with another PC), thus, local files are older - and these are not present remotely. Interpretation of change: You have delete these files - remotely, because your remotely files are newer that these files. - - delete these local files! (may ask the user) - - Download any new files - - Are remote files present, which are locally not present? - - Meaning: These files are new to the local side - - Download these files! (may ask the user) - - Upload any new files - - Are local files present, which are remotely not present? - - Meaning: These files are new to the remote side - - Upload these files! (may ask the user) - - Changed remote files - - Are remote and local files present, where the remote file is newer? - - Meaning: These files are changed on the remote side - - Download these files and replace the local copies (may ask the user) - - Changed local files - - Are local and remote files present, where the local file is newer? - - Meaning: These files are changed on the local side - - Upload these files and replace the remote copies (may ask the user) - */ - - // - // Read the local files ============================================================================================ - // - log.Println("Try to read all local files now...") - localFiles = make(map[string]os.FileInfo) - filepath.Walk(localDir, walkerlocal) - log.Printf("Found %d local files.\n", len(localFiles)) - - // - // Connect to the server ============================================================================================ - // - sftp, sftpError := sftp.NewClient(ssh) - if sftpError != nil { - log.Println("Was not able to connect to the server: " + sftpError.Error()) - return - } - - defer sftp.Close() - - // - // Read the remote files ============================================================================================ - // - log.Println("Try to read all remote files now...") - remoteFiles = make(map[string]os.FileInfo) - counterRemoteFile := 0 - walker := sftp.Walk(remoteDir) - for walker.Step() { - counterRemoteFile++ - if walker.Err() != nil { - continue - } - - remoteFiles[walker.Path()] = walker.Stat() - if counterRemoteFile%512 == 0 { - fmt.Printf("%05d.\n", counterRemoteFile) - } else if counterRemoteFile%16 == 0 { - fmt.Printf("%05d.", counterRemoteFile) - } - } - - fmt.Println() - log.Printf("Found %d remote files.\n", len(remoteFiles)) - - // - // Normalise all local paths ============================================================================================ - // - localFilesNormalised = make(map[string]os.FileInfo) - for key, value := range localFiles { - normalised := normalisePath(localDir, key) - if normalised != `` { - localFilesNormalised[normalised] = value - normalised2localFiles[normalised] = key - } - } - - // - // Normalise all remote paths ============================================================================================ - // - remoteFilesNormalised = make(map[string]os.FileInfo) - for key, value := range remoteFiles { - normalised := normalisePath(remoteDir, key) - if normalised != `` { - remoteFilesNormalised[normalised] = value - normalised2remoteFiles[normalised] = key - } - } - - // - // Determine ndl and ndr ============================================================================================ - // - ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC) - ndr := ndl - for _, remoteFileInfo := range remoteFiles { - if remoteFileInfo.ModTime().UTC().After(ndr) { - ndr = remoteFileInfo.ModTime().UTC() - } - } - - for _, localFileInfo := range localFiles { - if localFileInfo.ModTime().UTC().After(ndl) { - ndl = localFileInfo.ModTime().UTC() - } - } - - log.Printf("The newest local file's time: %v\n", ndl) - log.Printf("The newest remote file's time: %v\n", ndr) - - // - // Free remote space ============================================================================================ - // - deleteRemoteFiles := make([]string, 0) - for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { - localFileNormaliesed := localFilesNormalised[remoteFileNormalised] - if localFileNormaliesed == nil { - if remoteFileInfo.ModTime().UTC().Before(ndl) { - deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised) - } - } - } - - log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles)) - - if len(deleteRemoteFiles) > 0 { - sort.Strings(deleteRemoteFiles) - shouldDeleteRemoteFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range deleteRemoteFiles { - fmt.Println(normalised2remoteFiles[file]) - } - - fmt.Print("Should I delete these remote files? ") - shouldDeleteRemoteFiles = readYesNoAnswer(false) - } - - if shouldDeleteRemoteFiles { - // 1. Remove all files - for _, remoteFileNormalised := range deleteRemoteFiles { - - // Skip all directories: - if remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue - } - - removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised]) - if removeError != nil { - log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised]) - } - } - - // 2. Remove all directories - for _, remoteFileNormalised := range deleteRemoteFiles { - - // Skip all files: - if !remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue - } - - // TODO: Does not work: File an issue! - removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`) - if removeError != nil { - log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised]) - } - } - - for _, remoteFileNormalised := range deleteRemoteFiles { - delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised]) - delete(remoteFilesNormalised, remoteFileNormalised) - } - } - } - - // - // Free local space ============================================================================================ - // - if !pushOnly { - deleteLocalFiles := make([]string, 0) - for localFileNormalised, localFileInfo := range localFilesNormalised { - remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] - if remoteFileNormaliesed == nil { - if localFileInfo.ModTime().UTC().Before(ndr) { - deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) - } - } - } - - log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) - - if len(deleteLocalFiles) > 0 { - sort.Strings(deleteLocalFiles) - shouldDeleteLocalFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range deleteLocalFiles { - fmt.Println(normalised2localFiles[file]) - } - - fmt.Print("Should I delete these local files? ") - shouldDeleteLocalFiles = readYesNoAnswer(false) - } - - if shouldDeleteLocalFiles { - for _, localFileNormalised := range deleteLocalFiles { - - // Skip all directories: - if localFilesNormalised[localFileNormalised].IsDir() { - continue - } - - removeError := os.Remove(normalised2localFiles[localFileNormalised]) - if removeError != nil { - log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised]) - } - } - - for _, localFileNormalised := range deleteLocalFiles { - - // Skip all files: - if !localFilesNormalised[localFileNormalised].IsDir() { - continue - } - - removeError := os.Remove(normalised2localFiles[localFileNormalised]) - if removeError != nil { - log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) - } else { - log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised]) - } - } - - for _, localFileNormalised := range deleteLocalFiles { - delete(localFiles, normalised2localFiles[localFileNormalised]) - delete(localFilesNormalised, localFileNormalised) - } - } - } - } - - // - // Download new files ============================================================================================ - // - - if !pushOnly { - downloadRemoteFiles := make([]string, 0) - for remoteFileNormalised, _ := range remoteFilesNormalised { - localFileNormaliesed := localFilesNormalised[remoteFileNormalised] - if localFileNormaliesed == nil { - downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) - } - } - - log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles)) - - if len(downloadRemoteFiles) > 0 { - sort.Strings(downloadRemoteFiles) - shouldDownloadRemoteFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range downloadRemoteFiles { - fmt.Println(normalised2remoteFiles[file]) - } - - fmt.Print("Should I download these new remote files? ") - shouldDownloadRemoteFiles = readYesNoAnswer(false) - } - - if shouldDownloadRemoteFiles { - - // 1. Create all new directories - for _, remoteFileNormalised := range downloadRemoteFiles { - - // Skip all files - if !remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue - } - - newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - log.Printf("Try to create the new local directory %s...\n", newLocalDir) - if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil { - log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error()) - } - } - - // 2. All new files - for _, remoteFileNormalised := range downloadRemoteFiles { - - // Skip all directories - if remoteFilesNormalised[remoteFileNormalised].IsDir() { - continue - } - - log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) - remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) - if remoteFileHandleError != nil { - log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) - continue - } - - _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - newLocalFile := filepath.Join(localDir, path, filename) - localFileHandle, localFileHandleError := os.Create(newLocalFile) - if localFileHandleError != nil { - log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } - - _, copyError := io.Copy(localFileHandle, remoteFileHandle) - if copyError != nil { - log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error()) - remoteFileHandle.Close() - localFileHandle.Close() - continue - } - - remoteFileHandle.Close() - localFileHandle.Close() - } - } - } - } - - // - // Upload new files ============================================================================================ - // - - uploadLocalFiles := make([]string, 0) - for localFileNormalised, _ := range localFilesNormalised { - remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] - if remoteFileNormaliesed == nil { - uploadLocalFiles = append(uploadLocalFiles, localFileNormalised) - } - } - - log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles)) - - if len(uploadLocalFiles) > 0 { - sort.Strings(uploadLocalFiles) - shouldUploadLocalFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range uploadLocalFiles { - fmt.Println(normalised2localFiles[file]) - } - - fmt.Print("Should I upload these local new files? ") - shouldUploadLocalFiles = readYesNoAnswer(false) - } - - if shouldUploadLocalFiles { - - // 1. Create new directories - for _, localFileNormalised := range uploadLocalFiles { - - // Skip all files - if !localFilesNormalised[localFileNormalised].IsDir() { - continue - } - - newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))) - log.Printf("Try to create the new remote directory %s...\n", newRemoteDir) - if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil { - log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error()) - } - } - - // 2. All new files - for _, localFileNormalised := range uploadLocalFiles { - - // Skip all directories - if localFilesNormalised[localFileNormalised].IsDir() { - continue - } - - log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised]) - - _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) - newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) - remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile) - if remoteFileHandleError != nil { - log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error()) - continue - } - - localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) - if localFileHandleError != nil { - log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } - - _, copyError := io.Copy(remoteFileHandle, localFileHandle) - if copyError != nil { - log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error()) - remoteFileHandle.Close() - localFileHandle.Close() - continue - } - - remoteFileHandle.Close() - localFileHandle.Close() - } - } - } - - // - // Changed files on the remote side ============================================================================================ - // - - if !pushOnly { - changedRemoteFiles := make([]string, 0) - for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { - localFileNormaliesed := localFilesNormalised[remoteFileNormalised] - if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { - if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { - changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) - } - } - } - - log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) - - if len(changedRemoteFiles) > 0 { - sort.Strings(changedRemoteFiles) - shouldDownloadRemoteFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range changedRemoteFiles { - fmt.Println(normalised2remoteFiles[file]) - } - - fmt.Print("Should I download these changed remote files? ") - shouldDownloadRemoteFiles = readYesNoAnswer(false) - } - - if shouldDownloadRemoteFiles { - for _, remoteFileNormalised := range changedRemoteFiles { - log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) - remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) - if remoteFileHandleError != nil { - log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) - continue - } - - _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) - existingLocalFile := filepath.Join(localDir, path, filename) - localFileHandle, localFileHandleError := os.Create(existingLocalFile) - if localFileHandleError != nil { - log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } - - _, copyError := io.Copy(localFileHandle, remoteFileHandle) - if copyError != nil { - log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error()) - remoteFileHandle.Close() - localFileHandle.Close() - continue - } - - remoteFileHandle.Close() - localFileHandle.Close() - } - } - } - } - - // - // Changed files on the local side ============================================================================================ - // - - changedLocalFiles := make([]string, 0) - for localFileNormalised, localFileInfo := range localFilesNormalised { - remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] - if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() { - if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) { - changedLocalFiles = append(changedLocalFiles, localFileNormalised) - } - } - } - - log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles)) - - if len(changedLocalFiles) > 0 { - sort.Strings(changedLocalFiles) - shouldUploadLocalFiles := true - if supervised { - fmt.Println(`=================================================================`) - for _, file := range changedLocalFiles { - fmt.Println(normalised2localFiles[file]) - } - - fmt.Print("Should I upload these changed local files? ") - shouldUploadLocalFiles = readYesNoAnswer(false) - } - - if shouldUploadLocalFiles { - for _, localFileNormalised := range changedLocalFiles { - log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised]) - - _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) - path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) - existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) - remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile) - if remoteFileHandleError != nil { - log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error()) - continue - } - - localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) - if localFileHandleError != nil { - log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) - remoteFileHandle.Close() - continue - } - - _, copyError := io.Copy(remoteFileHandle, localFileHandle) - if copyError != nil { - log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error()) - remoteFileHandle.Close() - localFileHandle.Close() - continue - } - - remoteFileHandle.Close() - localFileHandle.Close() - } - } - } -} +package Sync + +import ( + "fmt" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) { + /* + Algorithm: + - Get all local files and dirs + - Get all remote files and dirs + - Determine: ndl := newest mod time of all local files + - Determine: ndr := newest mod time of all remote files + - Free remote space + - Are remote files present, which are locally not present + and these remote files are older than ndl? + - Meaning: You have worked locally and remotely are older files which are not locally present. + Interpretation of change: You have deleted these files locally, because you own local + files which are newer that these files. + - delete these remote files! (may ask the user) + - Free local space + - Are local files present, which are remotely not present + and these local files are older than ndr? + - Meaning: You have worked remotely (or have synced with another PC), thus, local files are older + and these are not present remotely. Interpretation of change: You have delete these files + remotely, because your remotely files are newer that these files. + - delete these local files! (may ask the user) + - Download any new files + - Are remote files present, which are locally not present? + - Meaning: These files are new to the local side + - Download these files! (may ask the user) + - Upload any new files + - Are local files present, which are remotely not present? + - Meaning: These files are new to the remote side + - Upload these files! (may ask the user) + - Changed remote files + - Are remote and local files present, where the remote file is newer? + - Meaning: These files are changed on the remote side + - Download these files and replace the local copies (may ask the user) + - Changed local files + - Are local and remote files present, where the local file is newer? + - Meaning: These files are changed on the local side + - Upload these files and replace the remote copies (may ask the user) + */ + + // + // Read the local files ============================================================================================ + // + log.Println("Try to read all local files now...") + localFiles = make(map[string]os.FileInfo) + filepath.Walk(localDir, walkerlocal) + log.Printf("Found %d local files.\n", len(localFiles)) + + // + // Connect to the server ============================================================================================ + // + sftp, sftpError := sftp.NewClient(ssh) + if sftpError != nil { + log.Println("Was not able to connect to the server: " + sftpError.Error()) + os.Exit(7) + return + } + + defer sftp.Close() + + // + // Read the remote files ============================================================================================ + // + log.Println("Try to read all remote files now...") + remoteFiles = make(map[string]os.FileInfo) + counterRemoteFile := 0 + walker := sftp.Walk(remoteDir) + for walker.Step() { + counterRemoteFile++ + if walker.Err() != nil { + continue + } + + remoteFiles[walker.Path()] = walker.Stat() + if counterRemoteFile%512 == 0 { + fmt.Printf("%05d.\n", counterRemoteFile) + } else if counterRemoteFile%16 == 0 { + fmt.Printf("%05d.", counterRemoteFile) + } + } + + fmt.Println() + log.Printf("Found %d remote files.\n", len(remoteFiles)) + + // + // Normalise all local paths ============================================================================================ + // + localFilesNormalised = make(map[string]os.FileInfo) + for key, value := range localFiles { + normalised := normalisePath(localDir, key) + if normalised != `` { + localFilesNormalised[normalised] = value + normalised2localFiles[normalised] = key + } + } + + // + // Normalise all remote paths ============================================================================================ + // + remoteFilesNormalised = make(map[string]os.FileInfo) + for key, value := range remoteFiles { + normalised := normalisePath(remoteDir, key) + if normalised != `` { + remoteFilesNormalised[normalised] = value + normalised2remoteFiles[normalised] = key + } + } + + // + // Determine ndl and ndr ============================================================================================ + // + ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC) + ndr := ndl + for _, remoteFileInfo := range remoteFiles { + if remoteFileInfo.ModTime().UTC().After(ndr) { + ndr = remoteFileInfo.ModTime().UTC() + } + } + + for _, localFileInfo := range localFiles { + if localFileInfo.ModTime().UTC().After(ndl) { + ndl = localFileInfo.ModTime().UTC() + } + } + + log.Printf("The newest local file's time: %v\n", ndl) + log.Printf("The newest remote file's time: %v\n", ndr) + + // + // Free remote space ============================================================================================ + // + deleteRemoteFiles := make([]string, 0) + for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed == nil { + if remoteFileInfo.ModTime().UTC().Before(ndl) { + deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised) + } + } + } + + log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles)) + + if len(deleteRemoteFiles) > 0 { + sort.Strings(deleteRemoteFiles) + shouldDeleteRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range deleteRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I delete these remote files? ") + shouldDeleteRemoteFiles = readYesNoAnswer(false) + } + + if shouldDeleteRemoteFiles { + // 1. Remove all files + for _, remoteFileNormalised := range deleteRemoteFiles { + + // Skip all directories: + if remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised]) + } + } + + // 2. Remove all directories + for _, remoteFileNormalised := range deleteRemoteFiles { + + // Skip all files: + if !remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + // TODO: Does not work: File an issue! + removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`) + if removeError != nil { + log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised]) + } + } + + for _, remoteFileNormalised := range deleteRemoteFiles { + delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised]) + delete(remoteFilesNormalised, remoteFileNormalised) + } + } + } + + // + // Free local space ============================================================================================ + // + if !pushOnly { + deleteLocalFiles := make([]string, 0) + for localFileNormalised, localFileInfo := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed == nil { + if localFileInfo.ModTime().UTC().Before(ndr) { + deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) + } + } + } + + log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) + + if len(deleteLocalFiles) > 0 { + sort.Strings(deleteLocalFiles) + shouldDeleteLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range deleteLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I delete these local files? ") + shouldDeleteLocalFiles = readYesNoAnswer(false) + } + + if shouldDeleteLocalFiles { + for _, localFileNormalised := range deleteLocalFiles { + + // Skip all directories: + if localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised]) + } + } + + for _, localFileNormalised := range deleteLocalFiles { + + // Skip all files: + if !localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised]) + } + } + + for _, localFileNormalised := range deleteLocalFiles { + delete(localFiles, normalised2localFiles[localFileNormalised]) + delete(localFilesNormalised, localFileNormalised) + } + } + } + } + + // + // Download new files ============================================================================================ + // + + if !pushOnly { + downloadRemoteFiles := make([]string, 0) + for remoteFileNormalised, _ := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed == nil { + downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) + } + } + + log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles)) + + if len(downloadRemoteFiles) > 0 { + sort.Strings(downloadRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range downloadRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I download these new remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) + } + + if shouldDownloadRemoteFiles { + + // 1. Create all new directories + for _, remoteFileNormalised := range downloadRemoteFiles { + + // Skip all files + if !remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + log.Printf("Try to create the new local directory %s...\n", newLocalDir) + if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil { + log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error()) + } + } + + // 2. All new files + for _, remoteFileNormalised := range downloadRemoteFiles { + + // Skip all directories + if remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } + + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + newLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(newLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + } + + // + // Upload new files ============================================================================================ + // + + uploadLocalFiles := make([]string, 0) + for localFileNormalised, _ := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed == nil { + uploadLocalFiles = append(uploadLocalFiles, localFileNormalised) + } + } + + log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles)) + + if len(uploadLocalFiles) > 0 { + sort.Strings(uploadLocalFiles) + shouldUploadLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range uploadLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I upload these local new files? ") + shouldUploadLocalFiles = readYesNoAnswer(false) + } + + if shouldUploadLocalFiles { + + // 1. Create new directories + for _, localFileNormalised := range uploadLocalFiles { + + // Skip all files + if !localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))) + log.Printf("Try to create the new remote directory %s...\n", newRemoteDir) + if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil { + log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error()) + } + } + + // 2. All new files + for _, localFileNormalised := range uploadLocalFiles { + + // Skip all directories + if localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised]) + + _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) + newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) + remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile) + if remoteFileHandleError != nil { + log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error()) + continue + } + + localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) + if localFileHandleError != nil { + log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(remoteFileHandle, localFileHandle) + if copyError != nil { + log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + + // + // Changed files on the remote side ============================================================================================ + // + + if !pushOnly { + changedRemoteFiles := make([]string, 0) + for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { + if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { + changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) + } + } + } + + log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) + + if len(changedRemoteFiles) > 0 { + sort.Strings(changedRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range changedRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I download these changed remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) + } + + if shouldDownloadRemoteFiles { + for _, remoteFileNormalised := range changedRemoteFiles { + log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } + + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + existingLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(existingLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + } + + // + // Changed files on the local side ============================================================================================ + // + + changedLocalFiles := make([]string, 0) + for localFileNormalised, localFileInfo := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() { + if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) { + changedLocalFiles = append(changedLocalFiles, localFileNormalised) + } + } + } + + log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles)) + + if len(changedLocalFiles) > 0 { + sort.Strings(changedLocalFiles) + shouldUploadLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range changedLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I upload these changed local files? ") + shouldUploadLocalFiles = readYesNoAnswer(false) + } + + if shouldUploadLocalFiles { + for _, localFileNormalised := range changedLocalFiles { + log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised]) + + _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) + existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) + remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile) + if remoteFileHandleError != nil { + log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error()) + continue + } + + localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) + if localFileHandleError != nil { + log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(remoteFileHandle, localFileHandle) + if copyError != nil { + log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } +}