From 4b7701ea0da91aa95caaebe849235bea90412fb9 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 29 Sep 2015 21:26:34 +0200 Subject: [PATCH] Version 1.0.0 --- Main.go | 14 ++ Sync/Synchronise.go | 552 +++++++++++++++++++++++++++++++++++++++++++- Sync/Variables.go | 14 +- 3 files changed, 574 insertions(+), 6 deletions(-) diff --git a/Main.go b/Main.go index 2cfed36..6b90c48 100644 --- a/Main.go +++ b/Main.go @@ -5,6 +5,7 @@ import ( "github.com/SommerEngineering/Sync/Sync" "golang.org/x/crypto/ssh" "log" + "os" "runtime" ) @@ -25,6 +26,17 @@ func main() { return } + // Check if local dir exist + if dirInfo, dirError := os.Stat(localDir); dirError != nil { + log.Println("There is an error with the local directory: " + dirError.Error()) + return + } else { + if !dirInfo.IsDir() { + log.Println("There is an error with the local directory: You provided a file instead!") + return + } + } + // Check if the password was provided: for true { if password == `` { @@ -55,4 +67,6 @@ func main() { } defer ssh.Close() + Sync.Synchronise(ssh, supervised, localDir, remoteDir) + log.Println("Synchronising done.") } diff --git a/Sync/Synchronise.go b/Sync/Synchronise.go index 2bf033a..4a1133a 100644 --- a/Sync/Synchronise.go +++ b/Sync/Synchronise.go @@ -1,8 +1,20 @@ package Sync import ( + "fmt" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +var ( + test string = "" ) func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { @@ -27,13 +39,11 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { remotely, because your remotely files are newer that these files. - delete these local files! (may ask the user) - Download any new files - - Are remote files present, which are locally not present - and these remote files are newer than ndl? + - Are remote files present, which are locally not present? - Meaning: These files are new to the local side - Download these files! (may ask the user) - Upload any new files - - Are local files present, which are remotely not present - and these local files are newer than ndr? + - Are local files present, which are remotely not present? - Meaning: These files are new to the remote side - Upload these files! (may ask the user) - Changed remote files @@ -45,4 +55,538 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) { - Meaning: These files are changed on the local side - Upload these files and replace the remote copies (may ask the user) */ + + // + // Read the local files ============================================================================================ + // + log.Println("Try to read all local files now...") + localFiles = make(map[string]os.FileInfo) + filepath.Walk(localDir, walkerlocal) + log.Printf("Found %d local files.\n", len(localFiles)) + + // + // Read the remote files ============================================================================================ + // + log.Println("Try to read all remote files now...") + remoteFiles = make(map[string]os.FileInfo) + + sftp, sftpError := sftp.NewClient(ssh) + if sftpError != nil { + log.Println("Was not able to connect to the server: " + sftpError.Error()) + return + } + + defer sftp.Close() + + counterRemoteFile := 0 + walker := sftp.Walk(remoteDir) + for walker.Step() { + counterRemoteFile++ + if walker.Err() != nil { + continue + } + + remoteFiles[walker.Path()] = walker.Stat() + if counterRemoteFile%512 == 0 { + fmt.Printf("%05d.\n", counterRemoteFile) + } else if counterRemoteFile%16 == 0 { + fmt.Printf("%05d.", counterRemoteFile) + } + } + + fmt.Println() + log.Printf("Found %d remote files.\n", len(remoteFiles)) + + // + // Normalise all local paths ============================================================================================ + // + localFilesNormalised = make(map[string]os.FileInfo) + for key, value := range localFiles { + normalised := normalisePath(localDir, key) + if normalised != `` { + localFilesNormalised[normalised] = value + normalised2localFiles[normalised] = key + } + } + + // + // Normalise all remote paths ============================================================================================ + // + remoteFilesNormalised = make(map[string]os.FileInfo) + for key, value := range remoteFiles { + normalised := normalisePath(remoteDir, key) + if normalised != `` { + remoteFilesNormalised[normalised] = value + normalised2remoteFiles[normalised] = key + } + } + + // + // Determine ndl and ndr ============================================================================================ + // + ndl := time.Date(1, time.January, 1, 1, 1, 1, 1, time.UTC) + ndr := ndl + for _, remoteFileInfo := range remoteFiles { + if remoteFileInfo.ModTime().UTC().After(ndr) { + ndr = remoteFileInfo.ModTime().UTC() + } + } + + for _, localFileInfo := range localFiles { + if localFileInfo.ModTime().UTC().After(ndl) { + ndl = localFileInfo.ModTime().UTC() + } + } + + log.Printf("The newest local file's time: %v\n", ndl) + log.Printf("The newest remote file's time: %v\n", ndr) + + // + // Free remote space ============================================================================================ + // + deleteRemoteFiles := make([]string, 0) + for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed == nil { + if remoteFileInfo.ModTime().UTC().Before(ndl) { + deleteRemoteFiles = append(deleteRemoteFiles, remoteFileNormalised) + } + } + } + + log.Printf("Found %d remote files to delete.\n", len(deleteRemoteFiles)) + + if len(deleteRemoteFiles) > 0 { + sort.Strings(deleteRemoteFiles) + shouldDeleteRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range deleteRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I delete these remote files? ") + shouldDeleteRemoteFiles = readYesNoAnswer(false) + } + + if shouldDeleteRemoteFiles { + // 1. Remove all files + for _, remoteFileNormalised := range deleteRemoteFiles { + + // Skip all directories: + if remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the remote file %s\n", normalised2remoteFiles[remoteFileNormalised]) + } + } + + // 2. Remove all directories + for _, remoteFileNormalised := range deleteRemoteFiles { + + // Skip all files: + if !remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + // TODO: Does not work: File an issue! + removeError := sftp.Remove(normalised2remoteFiles[remoteFileNormalised] + `/`) + if removeError != nil { + log.Printf("Was not able to delete the remote directory %s: %s\n", normalised2remoteFiles[remoteFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the remote directory %s\n", normalised2remoteFiles[remoteFileNormalised]) + } + } + + for _, remoteFileNormalised := range deleteRemoteFiles { + delete(remoteFiles, normalised2remoteFiles[remoteFileNormalised]) + delete(remoteFilesNormalised, remoteFileNormalised) + } + } + } + + // + // Free local space ============================================================================================ + // + deleteLocalFiles := make([]string, 0) + for localFileNormalised, localFileInfo := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed == nil { + if localFileInfo.ModTime().UTC().Before(ndr) { + deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) + } + } + } + + log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) + + if len(deleteLocalFiles) > 0 { + sort.Strings(deleteLocalFiles) + shouldDeleteLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range deleteLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I delete these local files? ") + shouldDeleteLocalFiles = readYesNoAnswer(false) + } + + if shouldDeleteLocalFiles { + for _, localFileNormalised := range deleteLocalFiles { + + // Skip all directories: + if localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised]) + } + } + + for _, localFileNormalised := range deleteLocalFiles { + + // Skip all files: + if !localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + removeError := os.Remove(normalised2localFiles[localFileNormalised]) + if removeError != nil { + log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) + } else { + log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised]) + } + } + + for _, localFileNormalised := range deleteLocalFiles { + delete(localFiles, normalised2localFiles[localFileNormalised]) + delete(localFilesNormalised, localFileNormalised) + } + } + } + + // + // Download new files ============================================================================================ + // + + downloadRemoteFiles := make([]string, 0) + for remoteFileNormalised, _ := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed == nil { + downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) + } + } + + log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles)) + + if len(downloadRemoteFiles) > 0 { + sort.Strings(downloadRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range downloadRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I download these new remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) + } + + if shouldDownloadRemoteFiles { + + // 1. Create all new directories + for _, remoteFileNormalised := range downloadRemoteFiles { + + // Skip all files + if !remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + log.Printf("Try to create the new local directory %s...\n", newLocalDir) + if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil { + log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error()) + } + } + + // 2. All new files + for _, remoteFileNormalised := range downloadRemoteFiles { + + // Skip all directories + if remoteFilesNormalised[remoteFileNormalised].IsDir() { + continue + } + + log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } + + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + newLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(newLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + + // + // Upload new files ============================================================================================ + // + + uploadLocalFiles := make([]string, 0) + for localFileNormalised, _ := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed == nil { + uploadLocalFiles = append(uploadLocalFiles, localFileNormalised) + } + } + + log.Printf("Found %d new local files to upload.\n", len(uploadLocalFiles)) + + if len(uploadLocalFiles) > 0 { + sort.Strings(uploadLocalFiles) + shouldUploadLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range uploadLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I upload these local new files? ") + shouldUploadLocalFiles = readYesNoAnswer(false) + } + + if shouldUploadLocalFiles { + + // 1. Create new directories + for _, localFileNormalised := range uploadLocalFiles { + + // Skip all files + if !localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + newRemoteDir := filepath.ToSlash(filepath.Join(remoteDir, strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1))) + log.Printf("Try to create the new remote directory %s...\n", newRemoteDir) + if mkdirError := sftp.Mkdir(newRemoteDir); mkdirError != nil { + log.Printf("Was not able to create the remote directory %s: %s\n", newRemoteDir, mkdirError.Error()) + } + } + + // 2. All new files + for _, localFileNormalised := range uploadLocalFiles { + + // Skip all directories + if localFilesNormalised[localFileNormalised].IsDir() { + continue + } + + log.Printf("Try to upload the new local file %s...\n", normalised2localFiles[localFileNormalised]) + + _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) + newRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) + remoteFileHandle, remoteFileHandleError := sftp.Create(newRemoteFile) + if remoteFileHandleError != nil { + log.Printf("Was not able to create the remote file %s: %s\n", newRemoteFile, remoteFileHandleError.Error()) + continue + } + + localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) + if localFileHandleError != nil { + log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(remoteFileHandle, localFileHandle) + if copyError != nil { + log.Printf("Was not able to upload the new local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], newRemoteFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + + // + // Changed files on the remote side ============================================================================================ + // + + changedRemoteFiles := make([]string, 0) + for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { + localFileNormaliesed := localFilesNormalised[remoteFileNormalised] + if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { + if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { + changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) + } + } + } + + log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) + + if len(changedRemoteFiles) > 0 { + sort.Strings(changedRemoteFiles) + shouldDownloadRemoteFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range changedRemoteFiles { + fmt.Println(normalised2remoteFiles[file]) + } + + fmt.Print("Should I download these changed remote files? ") + shouldDownloadRemoteFiles = readYesNoAnswer(false) + } + + if shouldDownloadRemoteFiles { + for _, remoteFileNormalised := range changedRemoteFiles { + log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) + remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) + if remoteFileHandleError != nil { + log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) + continue + } + + _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) + existingLocalFile := filepath.Join(localDir, path, filename) + localFileHandle, localFileHandleError := os.Create(existingLocalFile) + if localFileHandleError != nil { + log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(localFileHandle, remoteFileHandle) + if copyError != nil { + log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } + + // + // Changed files on the local side ============================================================================================ + // + + changedLocalFiles := make([]string, 0) + for localFileNormalised, localFileInfo := range localFilesNormalised { + remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] + if remoteFileNormaliesed != nil && !remoteFileNormaliesed.IsDir() { + if localFileInfo.ModTime().UTC().After(remoteFileNormaliesed.ModTime().UTC()) { + changedLocalFiles = append(changedLocalFiles, localFileNormalised) + } + } + } + + log.Printf("Found %d local files which are changed.\n", len(changedLocalFiles)) + + if len(changedLocalFiles) > 0 { + sort.Strings(changedLocalFiles) + shouldUploadLocalFiles := true + if supervised { + fmt.Println(`=================================================================`) + for _, file := range changedLocalFiles { + fmt.Println(normalised2localFiles[file]) + } + + fmt.Print("Should I upload these changed local files? ") + shouldUploadLocalFiles = readYesNoAnswer(false) + } + + if shouldUploadLocalFiles { + for _, localFileNormalised := range changedLocalFiles { + log.Printf("Try to upload the changed local file %s...\n", normalised2localFiles[localFileNormalised]) + + _, filename := filepath.Split(normalised2localFiles[localFileNormalised]) + path, _ := filepath.Split(strings.Replace(normalised2localFiles[localFileNormalised], localDir, ``, 1)) + existingRemoteFile := filepath.ToSlash(filepath.Join(remoteDir, path, filename)) + remoteFileHandle, remoteFileHandleError := sftp.Create(existingRemoteFile) + if remoteFileHandleError != nil { + log.Printf("Was not able to overwrite the remote file %s: %s\n", existingRemoteFile, remoteFileHandleError.Error()) + continue + } + + localFileHandle, localFileHandleError := os.Open(normalised2localFiles[localFileNormalised]) + if localFileHandleError != nil { + log.Printf("Was not able to open the local file %s: %s\n", normalised2localFiles[localFileNormalised], localFileHandleError.Error()) + remoteFileHandle.Close() + continue + } + + _, copyError := io.Copy(remoteFileHandle, localFileHandle) + if copyError != nil { + log.Printf("Was not able to upload the changed local file %s to the remote file %s: %s\n", normalised2localFiles[localFileNormalised], existingRemoteFile, copyError.Error()) + remoteFileHandle.Close() + localFileHandle.Close() + continue + } + + remoteFileHandle.Close() + localFileHandle.Close() + } + } + } +} + +func walkerlocal(path string, info os.FileInfo, err error) error { + localFiles[path] = info + return nil +} + +func readYesNoAnswer(defaultAnswer bool) bool { // true := yes + answer := `` + if _, scanError := fmt.Scan(&answer); scanError != nil { + return defaultAnswer + } + + if answer == `` || answer == ` ` { + return defaultAnswer + } + + if strings.ToLower(answer) == `y` { + return true + } + + return false } diff --git a/Sync/Variables.go b/Sync/Variables.go index 1c3b7df..13bee70 100644 --- a/Sync/Variables.go +++ b/Sync/Variables.go @@ -1,5 +1,15 @@ package Sync -var ( - callbackPassword = `` +import ( + "os" +) + +var ( + callbackPassword = `` + localFiles = make(map[string]os.FileInfo) // The local files with local separator + localFilesNormalised = make(map[string]os.FileInfo) // The local files with std separator i.e. / and removed base dir + normalised2localFiles = make(map[string]string) // Mapping from normalised local file to local file + remoteFiles = make(map[string]os.FileInfo) // The remote files with remote separator + remoteFilesNormalised = make(map[string]os.FileInfo) // The remote files with std separator i.e. / and removed base dir + normalised2remoteFiles = make(map[string]string) // Mapping from normalised remote file to remote file )