Version 1.0.0

This commit is contained in:
Thorsten Sommer 2015-09-30 10:51:22 +02:00
parent 4b7701ea0d
commit 815b15840c
10 changed files with 246 additions and 191 deletions

9
CorrectPath.go Normal file
View File

@ -0,0 +1,9 @@
package main
func correctPath(path string) string {
if path[len(path)-1:] == `/` || path[len(path)-1:] == `\` {
return path[:len(path)-1]
} else {
return path
}
}

34
Main.go
View File

@ -12,7 +12,7 @@ import (
func main() { func main() {
// Show the current version: // Show the current version:
fmt.Println(`Sync v1.0.0`) log.Println(`Sync v1.0.0`)
// Allow Go to use all CPUs: // Allow Go to use all CPUs:
runtime.GOMAXPROCS(runtime.NumCPU()) runtime.GOMAXPROCS(runtime.NumCPU())
@ -26,6 +26,21 @@ func main() {
return return
} }
// Should I use the current working dir?
if localDir == `.` {
if currentWD, currentWDError := os.Getwd(); currentWDError != nil {
log.Println("Cannot use the current working directory as local directory: " + currentWDError.Error())
return
} else {
log.Println("I use the current working directory as local directory: " + currentWD)
localDir = currentWD
}
}
// Remove trailing separators from both directories
localDir = correctPath(localDir)
remoteDir = correctPath(remoteDir)
// Check if local dir exist // Check if local dir exist
if dirInfo, dirError := os.Stat(localDir); dirError != nil { if dirInfo, dirError := os.Stat(localDir); dirError != nil {
log.Println("There is an error with the local directory: " + dirError.Error()) log.Println("There is an error with the local directory: " + dirError.Error())
@ -41,13 +56,26 @@ func main() {
for true { for true {
if password == `` { if password == `` {
// Promt for the password: // Promt for the password:
fmt.Println(`Please provide the password for the connection:`) fmt.Print(`Please provide the password for the connection: `)
fmt.Scanln(&password) fmt.Scanln(&password)
} else { } else {
break break
} }
} }
// Give some information about the state
if supervised {
log.Println("I use the supervised mode.")
} else {
log.Println("I do not use the supervised mode.")
}
if pushOnly {
log.Println("I use the push only mode i.e. backup mode. Any remote change will be ignored.")
} else {
log.Println("I use the full mode and consider also remote changes.")
}
// Create the SSH configuration: // Create the SSH configuration:
Sync.SetPassword4Callback(password) Sync.SetPassword4Callback(password)
config := &ssh.ClientConfig{ config := &ssh.ClientConfig{
@ -67,6 +95,6 @@ func main() {
} }
defer ssh.Close() defer ssh.Close()
Sync.Synchronise(ssh, supervised, localDir, remoteDir) Sync.Synchronise(ssh, supervised, pushOnly, localDir, remoteDir)
log.Println("Synchronising done.") log.Println("Synchronising done.")
} }

View File

@ -8,8 +8,9 @@ func readFlags() {
flag.StringVar(&serverAddrString, `server`, `127.0.0.1:22`, `The (remote) SSH server, e.g. 'my.host.com', 'my.host.com:22', '127.0.0.1:22', 'localhost:22'.`) flag.StringVar(&serverAddrString, `server`, `127.0.0.1:22`, `The (remote) SSH server, e.g. 'my.host.com', 'my.host.com:22', '127.0.0.1:22', 'localhost:22'.`)
flag.StringVar(&username, `user`, `username`, `The user's name for the SSH server.`) flag.StringVar(&username, `user`, `username`, `The user's name for the SSH server.`)
flag.StringVar(&password, `pwd`, ``, `The user's password for the SSH server. You can omit these argument. Thus, the program asks for the password on demand.`) flag.StringVar(&password, `pwd`, ``, `The user's password for the SSH server. You can omit these argument. Thus, the program asks for the password on demand.`)
flag.StringVar(&localDir, `localDir`, ``, `The local directory which should be synced.`) flag.StringVar(&localDir, `localDir`, `.`, `The local directory which should be synced. Use . for the current working directory.`)
flag.StringVar(&remoteDir, `remoteDir`, ``, `The remote directory which should be synced.`) flag.StringVar(&remoteDir, `remoteDir`, ``, `The remote directory which should be synced.`)
flag.BoolVar(&supervised, `supervised`, true, `Use the supervised mode? The algorithm asks you before any change.`) flag.BoolVar(&supervised, `supervised`, true, `Use the supervised mode? The algorithm asks you before any change.`)
flag.BoolVar(&pushOnly, `pushOnly`, true, `Use the push only mode, i.e. backup mode. Ignore any change on the remote side!`)
flag.Parse() flag.Parse()
} }

View File

@ -1,18 +1,7 @@
package Sync package Sync
import (
"path/filepath"
"strings"
)
func comparePath(localBase, localPath, remoteBase, remotePath string) bool { func comparePath(localBase, localPath, remoteBase, remotePath string) bool {
localCompare := normalisePath(localBase, localPath) localCompare := normalisePath(localBase, localPath)
remoteCompare := normalisePath(remoteBase, remotePath) remoteCompare := normalisePath(remoteBase, remotePath)
return localCompare == remoteCompare return localCompare == remoteCompare
} }
func normalisePath(base, path string) string {
result := strings.Replace(path, base, ``, 1)
result = filepath.ToSlash(result)
return strings.ToLower(result)
}

12
Sync/NormalisePath.go Normal file
View File

@ -0,0 +1,12 @@
package Sync
import (
"path/filepath"
"strings"
)
func normalisePath(base, path string) string {
result := strings.Replace(path, base, ``, 1)
result = filepath.ToSlash(result)
return strings.ToLower(result)
}

23
Sync/ReadYesNoAnswer.go Normal file
View File

@ -0,0 +1,23 @@
package Sync
import (
"fmt"
"strings"
)
func readYesNoAnswer(defaultAnswer bool) bool { // true := yes
answer := ``
if _, scanError := fmt.Scan(&answer); scanError != nil {
return defaultAnswer
}
if answer == `` || answer == ` ` {
return defaultAnswer
}
if strings.ToLower(answer) == `y` {
return true
}
return false
}

View File

@ -13,11 +13,7 @@ import (
"time" "time"
) )
var ( func Synchronise(ssh *ssh.Client, supervised, pushOnly bool, localDir, remoteDir string) {
test string = ""
)
func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
/* /*
Algorithm: Algorithm:
- Get all local files and dirs - Get all local files and dirs
@ -65,11 +61,8 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
log.Printf("Found %d local files.\n", len(localFiles)) log.Printf("Found %d local files.\n", len(localFiles))
// //
// Read the remote files ============================================================================================ // Connect to the server ============================================================================================
// //
log.Println("Try to read all remote files now...")
remoteFiles = make(map[string]os.FileInfo)
sftp, sftpError := sftp.NewClient(ssh) sftp, sftpError := sftp.NewClient(ssh)
if sftpError != nil { if sftpError != nil {
log.Println("Was not able to connect to the server: " + sftpError.Error()) log.Println("Was not able to connect to the server: " + sftpError.Error())
@ -78,6 +71,11 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
defer sftp.Close() defer sftp.Close()
//
// Read the remote files ============================================================================================
//
log.Println("Try to read all remote files now...")
remoteFiles = make(map[string]os.FileInfo)
counterRemoteFile := 0 counterRemoteFile := 0
walker := sftp.Walk(remoteDir) walker := sftp.Walk(remoteDir)
for walker.Step() { for walker.Step() {
@ -213,66 +211,68 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
// //
// Free local space ============================================================================================ // Free local space ============================================================================================
// //
deleteLocalFiles := make([]string, 0) if !pushOnly {
for localFileNormalised, localFileInfo := range localFilesNormalised { deleteLocalFiles := make([]string, 0)
remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised] for localFileNormalised, localFileInfo := range localFilesNormalised {
if remoteFileNormaliesed == nil { remoteFileNormaliesed := remoteFilesNormalised[localFileNormalised]
if localFileInfo.ModTime().UTC().Before(ndr) { if remoteFileNormaliesed == nil {
deleteLocalFiles = append(deleteLocalFiles, localFileNormalised) if localFileInfo.ModTime().UTC().Before(ndr) {
deleteLocalFiles = append(deleteLocalFiles, localFileNormalised)
}
} }
} }
}
log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles)) log.Printf("Found %d local files to delete.\n", len(deleteLocalFiles))
if len(deleteLocalFiles) > 0 { if len(deleteLocalFiles) > 0 {
sort.Strings(deleteLocalFiles) sort.Strings(deleteLocalFiles)
shouldDeleteLocalFiles := true shouldDeleteLocalFiles := true
if supervised { if supervised {
fmt.Println(`=================================================================`) fmt.Println(`=================================================================`)
for _, file := range deleteLocalFiles { for _, file := range deleteLocalFiles {
fmt.Println(normalised2localFiles[file]) fmt.Println(normalised2localFiles[file])
}
fmt.Print("Should I delete these local files? <y|n> ")
shouldDeleteLocalFiles = readYesNoAnswer(false)
} }
fmt.Print("Should I delete these local files? <y|n> ") if shouldDeleteLocalFiles {
shouldDeleteLocalFiles = readYesNoAnswer(false) for _, localFileNormalised := range deleteLocalFiles {
}
if shouldDeleteLocalFiles { // Skip all directories:
for _, localFileNormalised := range deleteLocalFiles { if localFilesNormalised[localFileNormalised].IsDir() {
continue
}
// Skip all directories: removeError := os.Remove(normalised2localFiles[localFileNormalised])
if localFilesNormalised[localFileNormalised].IsDir() { if removeError != nil {
continue log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised])
}
} }
removeError := os.Remove(normalised2localFiles[localFileNormalised]) for _, localFileNormalised := range deleteLocalFiles {
if removeError != nil {
log.Printf("Was not able to delete the local file %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the local file %s\n", normalised2localFiles[localFileNormalised])
}
}
for _, localFileNormalised := range deleteLocalFiles { // Skip all files:
if !localFilesNormalised[localFileNormalised].IsDir() {
continue
}
// Skip all files: removeError := os.Remove(normalised2localFiles[localFileNormalised])
if !localFilesNormalised[localFileNormalised].IsDir() { if removeError != nil {
continue log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error())
} else {
log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised])
}
} }
removeError := os.Remove(normalised2localFiles[localFileNormalised]) for _, localFileNormalised := range deleteLocalFiles {
if removeError != nil { delete(localFiles, normalised2localFiles[localFileNormalised])
log.Printf("Was not able to delete the local directory %s: %s\n", normalised2localFiles[localFileNormalised], removeError.Error()) delete(localFilesNormalised, localFileNormalised)
} else {
log.Printf("Deleted the local directory %s\n", normalised2localFiles[localFileNormalised])
} }
} }
for _, localFileNormalised := range deleteLocalFiles {
delete(localFiles, normalised2localFiles[localFileNormalised])
delete(localFilesNormalised, localFileNormalised)
}
} }
} }
@ -280,81 +280,83 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
// Download new files ============================================================================================ // Download new files ============================================================================================
// //
downloadRemoteFiles := make([]string, 0) if !pushOnly {
for remoteFileNormalised, _ := range remoteFilesNormalised { downloadRemoteFiles := make([]string, 0)
localFileNormaliesed := localFilesNormalised[remoteFileNormalised] for remoteFileNormalised, _ := range remoteFilesNormalised {
if localFileNormaliesed == nil { localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised) if localFileNormaliesed == nil {
} downloadRemoteFiles = append(downloadRemoteFiles, remoteFileNormalised)
}
log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles))
if len(downloadRemoteFiles) > 0 {
sort.Strings(downloadRemoteFiles)
shouldDownloadRemoteFiles := true
if supervised {
fmt.Println(`=================================================================`)
for _, file := range downloadRemoteFiles {
fmt.Println(normalised2remoteFiles[file])
} }
fmt.Print("Should I download these new remote files? <y|n> ")
shouldDownloadRemoteFiles = readYesNoAnswer(false)
} }
if shouldDownloadRemoteFiles { log.Printf("Found %d new remote files to download.\n", len(downloadRemoteFiles))
// 1. Create all new directories if len(downloadRemoteFiles) > 0 {
for _, remoteFileNormalised := range downloadRemoteFiles { sort.Strings(downloadRemoteFiles)
shouldDownloadRemoteFiles := true
// Skip all files if supervised {
if !remoteFilesNormalised[remoteFileNormalised].IsDir() { fmt.Println(`=================================================================`)
continue for _, file := range downloadRemoteFiles {
fmt.Println(normalised2remoteFiles[file])
} }
newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) fmt.Print("Should I download these new remote files? <y|n> ")
log.Printf("Try to create the new local directory %s...\n", newLocalDir) shouldDownloadRemoteFiles = readYesNoAnswer(false)
if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil {
log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error())
}
} }
// 2. All new files if shouldDownloadRemoteFiles {
for _, remoteFileNormalised := range downloadRemoteFiles {
// Skip all directories // 1. Create all new directories
if remoteFilesNormalised[remoteFileNormalised].IsDir() { for _, remoteFileNormalised := range downloadRemoteFiles {
continue
// Skip all files
if !remoteFilesNormalised[remoteFileNormalised].IsDir() {
continue
}
newLocalDir := filepath.Join(localDir, strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
log.Printf("Try to create the new local directory %s...\n", newLocalDir)
if mkdirError := os.MkdirAll(newLocalDir, os.ModeDir); mkdirError != nil {
log.Printf("Was not able to create the local directory %s: %s\n", newLocalDir, mkdirError.Error())
}
} }
log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) // 2. All new files
remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) for _, remoteFileNormalised := range downloadRemoteFiles {
if remoteFileHandleError != nil {
log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
continue
}
_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) // Skip all directories
path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) if remoteFilesNormalised[remoteFileNormalised].IsDir() {
newLocalFile := filepath.Join(localDir, path, filename) continue
localFileHandle, localFileHandleError := os.Create(newLocalFile) }
if localFileHandleError != nil {
log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error()) log.Printf("Try to download the new remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
remoteFileHandle.Close() remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
continue if remoteFileHandleError != nil {
} log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
continue
}
_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
newLocalFile := filepath.Join(localDir, path, filename)
localFileHandle, localFileHandleError := os.Create(newLocalFile)
if localFileHandleError != nil {
log.Printf("Was not able to create the local file %s: %s\n", newLocalFile, localFileHandleError.Error())
remoteFileHandle.Close()
continue
}
_, copyError := io.Copy(localFileHandle, remoteFileHandle)
if copyError != nil {
log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error())
remoteFileHandle.Close()
localFileHandle.Close()
continue
}
_, copyError := io.Copy(localFileHandle, remoteFileHandle)
if copyError != nil {
log.Printf("Was not able to download the new remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], newLocalFile, copyError.Error())
remoteFileHandle.Close() remoteFileHandle.Close()
localFileHandle.Close() localFileHandle.Close()
continue
} }
remoteFileHandle.Close()
localFileHandle.Close()
} }
} }
} }
@ -447,60 +449,62 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
// Changed files on the remote side ============================================================================================ // Changed files on the remote side ============================================================================================
// //
changedRemoteFiles := make([]string, 0) if !pushOnly {
for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised { changedRemoteFiles := make([]string, 0)
localFileNormaliesed := localFilesNormalised[remoteFileNormalised] for remoteFileNormalised, remoteFileInfo := range remoteFilesNormalised {
if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() { localFileNormaliesed := localFilesNormalised[remoteFileNormalised]
if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) { if localFileNormaliesed != nil && !localFileNormaliesed.IsDir() {
changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised) if remoteFileInfo.ModTime().UTC().After(localFileNormaliesed.ModTime().UTC()) {
changedRemoteFiles = append(changedRemoteFiles, remoteFileNormalised)
}
} }
} }
}
log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles)) log.Printf("Found %d remote files which are changed.\n", len(changedRemoteFiles))
if len(changedRemoteFiles) > 0 { if len(changedRemoteFiles) > 0 {
sort.Strings(changedRemoteFiles) sort.Strings(changedRemoteFiles)
shouldDownloadRemoteFiles := true shouldDownloadRemoteFiles := true
if supervised { if supervised {
fmt.Println(`=================================================================`) fmt.Println(`=================================================================`)
for _, file := range changedRemoteFiles { for _, file := range changedRemoteFiles {
fmt.Println(normalised2remoteFiles[file]) fmt.Println(normalised2remoteFiles[file])
}
fmt.Print("Should I download these changed remote files? <y|n> ")
shouldDownloadRemoteFiles = readYesNoAnswer(false)
} }
fmt.Print("Should I download these changed remote files? <y|n> ") if shouldDownloadRemoteFiles {
shouldDownloadRemoteFiles = readYesNoAnswer(false) for _, remoteFileNormalised := range changedRemoteFiles {
} log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised])
remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised])
if remoteFileHandleError != nil {
log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error())
continue
}
if shouldDownloadRemoteFiles { _, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised])
for _, remoteFileNormalised := range changedRemoteFiles { path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1))
log.Printf("Try to download the changed remote file %s...\n", normalised2remoteFiles[remoteFileNormalised]) existingLocalFile := filepath.Join(localDir, path, filename)
remoteFileHandle, remoteFileHandleError := sftp.Open(normalised2remoteFiles[remoteFileNormalised]) localFileHandle, localFileHandleError := os.Create(existingLocalFile)
if remoteFileHandleError != nil { if localFileHandleError != nil {
log.Printf("Was not able to open the remote file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], remoteFileHandleError.Error()) log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error())
continue remoteFileHandle.Close()
} continue
}
_, filename := filepath.Split(normalised2remoteFiles[remoteFileNormalised]) _, copyError := io.Copy(localFileHandle, remoteFileHandle)
path, _ := filepath.Split(strings.Replace(normalised2remoteFiles[remoteFileNormalised], remoteDir, ``, 1)) if copyError != nil {
existingLocalFile := filepath.Join(localDir, path, filename) log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error())
localFileHandle, localFileHandleError := os.Create(existingLocalFile) remoteFileHandle.Close()
if localFileHandleError != nil { localFileHandle.Close()
log.Printf("Was not able to overwrite the local file %s: %s\n", existingLocalFile, localFileHandleError.Error()) continue
remoteFileHandle.Close() }
continue
}
_, copyError := io.Copy(localFileHandle, remoteFileHandle)
if copyError != nil {
log.Printf("Was not able to download the changed remote file %s to the local file %s: %s\n", normalised2remoteFiles[remoteFileNormalised], existingLocalFile, copyError.Error())
remoteFileHandle.Close() remoteFileHandle.Close()
localFileHandle.Close() localFileHandle.Close()
continue
} }
remoteFileHandle.Close()
localFileHandle.Close()
} }
} }
} }
@ -568,25 +572,3 @@ func Synchronise(ssh *ssh.Client, supervised bool, localDir, remoteDir string) {
} }
} }
} }
func walkerlocal(path string, info os.FileInfo, err error) error {
localFiles[path] = info
return nil
}
func readYesNoAnswer(defaultAnswer bool) bool { // true := yes
answer := ``
if _, scanError := fmt.Scan(&answer); scanError != nil {
return defaultAnswer
}
if answer == `` || answer == ` ` {
return defaultAnswer
}
if strings.ToLower(answer) == `y` {
return true
}
return false
}

10
Sync/WalkerLocal.go Normal file
View File

@ -0,0 +1,10 @@
package Sync
import (
"os"
)
func walkerlocal(path string, info os.FileInfo, err error) error {
localFiles[path] = info
return nil
}

View File

@ -7,4 +7,5 @@ var (
localDir = `` // The local directory localDir = `` // The local directory
remoteDir = `` // The remote directory remoteDir = `` // The remote directory
supervised = true // Should the tool work supervised? supervised = true // Should the tool work supervised?
pushOnly = true // Pushes only local changes to the remote i.e. backup mode
) )