Go/Ocean
Go/Ocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved the ICCC

Browse Source 
The ICCC was improved to increase the performance and the accuracy for
floating point numbers. The efficiency of arrays is now increased caused
by these optimisations. Important: The new implementation is not
compatible with previous versions.
This commit is contained in:
Thorsten Sommer 2015-07-07 09:10:03 +02:00
parent 598f9b0ec0
commit ae68804bac
3 changed files with 340 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

273
ICCC/Data2Message.go
View File

@ -1,11 +1,15 @@
package ICCC
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"net/url"
"reflect"
"strconv"
"strings"
)
// Function to convert the HTTP data back to a message.
@ -49,37 +53,131 @@ func Data2Message(target interface{}, data map[string][]string) (channel, comman
// Use the order of the destination type's fields:
for i := 0; i < element.NumField(); i++ {
field := element.Field(i)
switch field.Kind().String() {
// Read the current field:
field := element.Field(i)
// Choose the right type for this field:
switch field.Kind().String() {
case `int64`:
// The name of the field:
mapName := fmt.Sprintf(`int:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseInt(mapValue, 10, 64)
field.SetInt(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination int64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
field.SetInt(destination)
}
}
case `string`:
// The name of the field:
mapName := fmt.Sprintf(`str:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
field.SetString(mapValue)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Decode the bytes as string:
text := string(bytesArray)
// Decode the URL encoded string:
textFinal, errURL := url.QueryUnescape(text)
if errURL != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode a URL encoded string.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errURL.Error())
} else {
field.SetString(textFinal)
}
}
case `float64`:
// The name of the field:
mapName := fmt.Sprintf(`f64:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseFloat(mapValue, 64)
field.SetFloat(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination float64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
field.SetFloat(destination)
}
}
case `bool`:
// The name of the field:
mapName := fmt.Sprintf(`bool:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseBool(mapValue)
field.SetBool(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the value:
if bytesArray[0] == 0x1 {
field.SetBool(true)
} else {
field.SetBool(false)
}
}
case `uint8`:
// The name of the field:
mapName := fmt.Sprintf(`ui8:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseUint(mapValue, 16, 8)
field.SetUint(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the value:
field.SetUint(uint64(bytesArray[0]))
}
// Case: Arrays...
case `slice`:
@ -87,64 +185,147 @@ func Data2Message(target interface{}, data map[string][]string) (channel, comman
sliceKind := reflect.ValueOf(sliceInterface).Type().String()
switch sliceKind {
case `[]uint8`: // bytes
case `[]uint8`: // a byte array
// The name of the field:
mapName := fmt.Sprintf(`ui8[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]uint8, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseUint(mapValue, 16, 8)
fieldData[n] = byte(v)
}
fieldDataValue := reflect.ValueOf(fieldData)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the values in the message:
fieldDataValue := reflect.ValueOf(bytesArray)
field.Set(fieldDataValue)
}
case `[]int64`:
// The name of the field:
mapName := fmt.Sprintf(`int[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]int64, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseInt(mapValue, 10, 64)
fieldData[n] = v
}
fieldDataValue := reflect.ValueOf(fieldData)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination []int64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
fieldDataValue := reflect.ValueOf(destination)
field.Set(fieldDataValue)
}
}
case `[]bool`:
// The name of the field:
mapName := fmt.Sprintf(`bool[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
fieldLen := len(bytesArray)
fieldData := make([]bool, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseBool(mapValue)
fieldData[n] = v
// Convert each byte in a bool:
for n, value := range bytesArray {
if value == 0x1 {
fieldData[n] = true
} else {
fieldData[n] = false
}
}
// Store the values in the message:
fieldDataValue := reflect.ValueOf(fieldData)
field.Set(fieldDataValue)
}
case `[]string`:
// The name of the field:
mapName := fmt.Sprintf(`str[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldDataValue := reflect.ValueOf(mapValues)
field.Set(fieldDataValue)
case `[]float64`:
mapName := fmt.Sprintf(`f64[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]float64, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseFloat(mapValue, 64)
fieldData[n] = v
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Get the URL encoded string of all values:
allStringsRAW := string(bytesArray)
// Split now the different strings:
allStrings := strings.Split(allStringsRAW, "\n")
// A place where we store the final strings:
data := make([]string, len(allStrings), len(allStrings))
// Loop over all URL encoded strings and decode it:
for n, element := range allStrings {
elementFinal, errURL := url.QueryUnescape(element)
if errURL != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode a base64 string for a string array.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errURL.Error())
} else {
data[n] = elementFinal
}
}
fieldDataValue := reflect.ValueOf(fieldData)
// Store the values in the message:
fieldDataValue := reflect.ValueOf(data)
field.Set(fieldDataValue)
}
case `[]float64`:
// The name of the field:
mapName := fmt.Sprintf(`f64[]:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination []float64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
fieldDataValue := reflect.ValueOf(destination)
field.Set(fieldDataValue)
}
}
}
}
}

27
ICCC/Doc.go
View File

@ -7,9 +7,9 @@ To be able to marshal / parse the data back to objects, some additional informat
Example 01:
name=str:Surname
value=Sommer
value=U29tbWVy
The HTTP form name is 'str:Surname' and the value is 'Sommer'. The 'str' is the indicator for the data type, in this case it is a string.
The HTTP form name is 'str:Surname' and the value is 'Sommer' as base64 encoded string. The 'str' is the indicator for the data type, in this case it is a string.
Known data types are:
* str := string
@ -23,18 +23,17 @@ Known data types are:
* str[] := string array
* f64[] := 64 bit float array
Formatting of the corresponding values (each value is a string => HTTP). Plase note:
For the arrays, the name will repeated for each value.
* str := the plain UTF8 string
* int := the integer e.g. '13894612'
* f64 := the float with nine digits e.g. 9.48 gets '9.480000000'
* bool := 'true' or 'false' (lower case)
* ui8 := the byte as hexadecimal string e.g. 255 gets 'ff'
* ui8[] := the bytes as hexdecimal strings e.g. 0 255 0 gets ui8[]:name:00 ui8[]:name:ff ui8[]:name:00
* int[] := 64 bit integer array e.g. 1 2 gets int[]:name:1 int[]:name:2
* bool[] := a boolean array e.g. true true gets bool[]:name:true bool[]:name:true
* str[] := string array e.g. 'a' 'abc' gets str[]:name:a str[]:name:abc
* f64[] := 64 bit float array e.g. 1.1 1.2 gets f64[]:name:1.100000000 f64[]:name:1.2000000000
Formatting of the corresponding values (each value is at the end a base64 string).
* str := the plain UTF8 string as URL encoded. These bytes are getting base64 encoded.
* int := the little endian representation of the int. These bytes are getting base64 encoded.
* f64 := the little endian representation of the float. These bytes are getting base64 encoded.
* bool := the byte 0x1 or 0x0 for true and false. These byte will be base64 encoded.
* ui8 := These byte will be base64 encoded.
* ui8[] := These bytes are getting base64 encoded.
* int[] := the little endian representation of the integers. These bytes are getting base64 encoded.
* bool[] := the bools are getting converted to bytes (0x1 or 0x0 for true and false). These bytes are getting base64 encoded.
* str[] := each string will be URL encoded. Afterwards, join all strings by \n. These bytes are getting base64 encoded.
* f64[] := the little endian representation of the floats. These bytes are getting base64 encoded.
The format of a message is:
command=COMMAND

120
ICCC/Message2Data.go
View File

@ -1,11 +1,16 @@
package ICCC
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"io"
"net/url"
"reflect"
"strconv"
"strings"
)
// Function to convert an ICCC message to HTTP data.
@ -39,74 +44,133 @@ func Message2Data(channel, command string, message interface{}) (data map[string
field := element.Field(i)
keyName := elementType.Field(i).Name
// A buffer for the binary representation:
buffer := new(bytes.Buffer)
// For possible errors:
var errConverter error = nil
// The key for this element:
key := ``
// Look for the right data type:
switch field.Kind().String() {
case `int64`:
key := fmt.Sprintf(`int:%s`, keyName)
data[key] = []string{strconv.FormatInt(field.Int(), 10)}
key = fmt.Sprintf(`int:%s`, keyName)
// Converts the value in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, field.Int())
case `string`:
key := fmt.Sprintf(`str:%s`, keyName)
data[key] = []string{field.String()}
key = fmt.Sprintf(`str:%s`, keyName)
// URL encode the string and copy its bytes to the buffer:
io.Copy(buffer, strings.NewReader(url.QueryEscape(field.String())))
case `float64`:
key := fmt.Sprintf(`f64:%s`, keyName)
data[key] = []string{strconv.FormatFloat(field.Float(), 'f', 9, 64)}
key = fmt.Sprintf(`f64:%s`, keyName)
// Converts the value in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, field.Float())
case `bool`:
key := fmt.Sprintf(`bool:%s`, keyName)
data[key] = []string{strconv.FormatBool(field.Bool())}
key = fmt.Sprintf(`bool:%s`, keyName)
case `uint8`: // byte
key := fmt.Sprintf(`ui8:%s`, keyName)
data[key] = []string{strconv.FormatUint(field.Uint(), 16)}
// Directly convert the bool in a byte:
if field.Bool() {
// Case: True
buffer.WriteByte(0x1) // Write 1
} else {
// Case: False
buffer.WriteByte(0x0) // Write 0
}
case `uint8`: // a byte
key = fmt.Sprintf(`ui8:%s`, keyName)
// uint8 is a byte, thus, write it directly in the buffer:
buffer.WriteByte(byte(field.Uint()))
// Case: Arrays...
case `slice`:
sliceLen := field.Len()
if sliceLen > 0 {
// Which kind of data is this?
sliceKind := field.Index(0).Kind()
key := ``
dataValues := make([]string, sliceLen, sliceLen)
// Select the right data type:
switch sliceKind.String() {
case `uint8`: // bytes
case `uint8`: // a byte array
key = fmt.Sprintf(`ui8[]:%s`, keyName)
values := field.Interface().([]uint8)
for index, value := range values {
dataValues[index] = strconv.FormatUint(uint64(value), 16)
// Directly write the bytes in the buffer:
for _, val := range values {
buffer.WriteByte(byte(val))
}
case `int64`:
key = fmt.Sprintf(`int[]:%s`, keyName)
values := field.Interface().([]int64)
for index, value := range values {
dataValues[index] = strconv.FormatInt(value, 10)
}
// Converts the array in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, values)
case `bool`:
key = fmt.Sprintf(`bool[]:%s`, keyName)
values := field.Interface().([]bool)
for index, value := range values {
dataValues[index] = strconv.FormatBool(value)
// Cannot convert bool to bytes by using binary.Write(). Thus,
// convert it by ower own:
// Loop over all values:
for _, val := range values {
if val {
// If the value is true:
buffer.WriteByte(0x1) // Write 1
} else {
// If the value is false:
buffer.WriteByte(0x0) // Write 0
}
}
case `string`:
key = fmt.Sprintf(`str[]:%s`, keyName)
values := field.Interface().([]string)
for index, value := range values {
dataValues[index] = value
// Mask every string by using a URL encoding.
// This masks e.g. every new-line i.e \n, etc.
// This allows us to combine later the strings by
// using \n:
masked := make([]string, len(values))
// Loop over each string and convert it:
for n, val := range values {
masked[n] = url.QueryEscape(val)
}
// Join all masked strings by using \n and copy the byte array
// representation in the buffer:
io.Copy(buffer, strings.NewReader(strings.Join(masked, "\n")))
case `float64`:
key = fmt.Sprintf(`f64[]:%s`, keyName)
values := field.Interface().([]float64)
for index, value := range values {
dataValues[index] = strconv.FormatFloat(value, 'f', 9, 64)
// Converts the array in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, values)
}
}
}
data[key] = dataValues
}
if errConverter != nil {
// An error occurs:
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `It was not possible to convert an array for an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errConverter.Error())
} else {
// Convert the byte array to a base64 string for the transportation on wire:
data[key] = []string{base64.StdEncoding.EncodeToString(buffer.Bytes())}
}
}