Improved the ICCC

The ICCC was improved to increase the performance and the accuracy for
floating point numbers. The efficiency of arrays is now increased caused
by these optimisations. Important: The new implementation is not
compatible with previous versions.
This commit is contained in:
Thorsten Sommer 2015-07-07 09:10:03 +02:00
parent 598f9b0ec0
commit ae68804bac
3 changed files with 340 additions and 96 deletions

View File

@ -1,11 +1,15 @@
package ICCC
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"net/url"
"reflect"
"strconv"
"strings"
)
// Function to convert the HTTP data back to a message.
@ -49,37 +53,131 @@ func Data2Message(target interface{}, data map[string][]string) (channel, comman
// Use the order of the destination type's fields:
for i := 0; i < element.NumField(); i++ {
field := element.Field(i)
switch field.Kind().String() {
// Read the current field:
field := element.Field(i)
// Choose the right type for this field:
switch field.Kind().String() {
case `int64`:
// The name of the field:
mapName := fmt.Sprintf(`int:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseInt(mapValue, 10, 64)
field.SetInt(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination int64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
field.SetInt(destination)
}
}
case `string`:
// The name of the field:
mapName := fmt.Sprintf(`str:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
field.SetString(mapValue)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Decode the bytes as string:
text := string(bytesArray)
// Decode the URL encoded string:
textFinal, errURL := url.QueryUnescape(text)
if errURL != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode a URL encoded string.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errURL.Error())
} else {
field.SetString(textFinal)
}
}
case `float64`:
// The name of the field:
mapName := fmt.Sprintf(`f64:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseFloat(mapValue, 64)
field.SetFloat(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination float64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
field.SetFloat(destination)
}
}
case `bool`:
// The name of the field:
mapName := fmt.Sprintf(`bool:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseBool(mapValue)
field.SetBool(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the value:
if bytesArray[0] == 0x1 {
field.SetBool(true)
} else {
field.SetBool(false)
}
}
case `uint8`:
// The name of the field:
mapName := fmt.Sprintf(`ui8:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
v, _ := strconv.ParseUint(mapValue, 16, 8)
field.SetUint(v)
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the value:
field.SetUint(uint64(bytesArray[0]))
}
// Case: Arrays...
case `slice`:
@ -87,64 +185,147 @@ func Data2Message(target interface{}, data map[string][]string) (channel, comman
sliceKind := reflect.ValueOf(sliceInterface).Type().String()
switch sliceKind {
case `[]uint8`: // bytes
case `[]uint8`: // a byte array
// The name of the field:
mapName := fmt.Sprintf(`ui8[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]uint8, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseUint(mapValue, 16, 8)
fieldData[n] = byte(v)
}
fieldDataValue := reflect.ValueOf(fieldData)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Store the values in the message:
fieldDataValue := reflect.ValueOf(bytesArray)
field.Set(fieldDataValue)
}
case `[]int64`:
// The name of the field:
mapName := fmt.Sprintf(`int[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]int64, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseInt(mapValue, 10, 64)
fieldData[n] = v
}
fieldDataValue := reflect.ValueOf(fieldData)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination []int64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
fieldDataValue := reflect.ValueOf(destination)
field.Set(fieldDataValue)
}
}
case `[]bool`:
// The name of the field:
mapName := fmt.Sprintf(`bool[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
fieldLen := len(bytesArray)
fieldData := make([]bool, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseBool(mapValue)
fieldData[n] = v
// Convert each byte in a bool:
for n, value := range bytesArray {
if value == 0x1 {
fieldData[n] = true
} else {
fieldData[n] = false
}
}
// Store the values in the message:
fieldDataValue := reflect.ValueOf(fieldData)
field.Set(fieldDataValue)
}
case `[]string`:
// The name of the field:
mapName := fmt.Sprintf(`str[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldDataValue := reflect.ValueOf(mapValues)
field.Set(fieldDataValue)
case `[]float64`:
mapName := fmt.Sprintf(`f64[]:%s`, elementType.Field(i).Name)
mapValues := data[mapName]
fieldLen := len(mapValues)
fieldData := make([]float64, fieldLen, fieldLen)
for n, mapValue := range mapValues {
v, _ := strconv.ParseFloat(mapValue, 64)
fieldData[n] = v
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// Get the URL encoded string of all values:
allStringsRAW := string(bytesArray)
// Split now the different strings:
allStrings := strings.Split(allStringsRAW, "\n")
// A place where we store the final strings:
data := make([]string, len(allStrings), len(allStrings))
// Loop over all URL encoded strings and decode it:
for n, element := range allStrings {
elementFinal, errURL := url.QueryUnescape(element)
if errURL != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode a base64 string for a string array.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errURL.Error())
} else {
data[n] = elementFinal
}
}
fieldDataValue := reflect.ValueOf(fieldData)
// Store the values in the message:
fieldDataValue := reflect.ValueOf(data)
field.Set(fieldDataValue)
}
case `[]float64`:
// The name of the field:
mapName := fmt.Sprintf(`f64[]:%s`, elementType.Field(i).Name)
// The value of the field as string:
mapValue := data[mapName][0]
// The value of the field as bytes:
bytesArray, errBase64 := base64.StdEncoding.DecodeString(mapValue)
if errBase64 != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the base64 data to an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBase64.Error())
} else {
// The destination:
var destination []float64
// A reader for the bytes:
buffer := bytes.NewReader(bytesArray)
// Try to decode the bytes to an instance of the type:
errBinary := binary.Read(buffer, binary.LittleEndian, &destination)
if errBinary != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `Was not able to decode the binary data to the type of the ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errBinary.Error())
} else {
// Finnaly, store the value in the message:
fieldDataValue := reflect.ValueOf(destination)
field.Set(fieldDataValue)
}
}
}
}
}

View File

@ -7,9 +7,9 @@ To be able to marshal / parse the data back to objects, some additional informat
Example 01:
name=str:Surname
value=Sommer
value=U29tbWVy
The HTTP form name is 'str:Surname' and the value is 'Sommer'. The 'str' is the indicator for the data type, in this case it is a string.
The HTTP form name is 'str:Surname' and the value is 'Sommer' as base64 encoded string. The 'str' is the indicator for the data type, in this case it is a string.
Known data types are:
* str := string
@ -23,18 +23,17 @@ Known data types are:
* str[] := string array
* f64[] := 64 bit float array
Formatting of the corresponding values (each value is a string => HTTP). Plase note:
For the arrays, the name will repeated for each value.
* str := the plain UTF8 string
* int := the integer e.g. '13894612'
* f64 := the float with nine digits e.g. 9.48 gets '9.480000000'
* bool := 'true' or 'false' (lower case)
* ui8 := the byte as hexadecimal string e.g. 255 gets 'ff'
* ui8[] := the bytes as hexdecimal strings e.g. 0 255 0 gets ui8[]:name:00 ui8[]:name:ff ui8[]:name:00
* int[] := 64 bit integer array e.g. 1 2 gets int[]:name:1 int[]:name:2
* bool[] := a boolean array e.g. true true gets bool[]:name:true bool[]:name:true
* str[] := string array e.g. 'a' 'abc' gets str[]:name:a str[]:name:abc
* f64[] := 64 bit float array e.g. 1.1 1.2 gets f64[]:name:1.100000000 f64[]:name:1.2000000000
Formatting of the corresponding values (each value is at the end a base64 string).
* str := the plain UTF8 string as URL encoded. These bytes are getting base64 encoded.
* int := the little endian representation of the int. These bytes are getting base64 encoded.
* f64 := the little endian representation of the float. These bytes are getting base64 encoded.
* bool := the byte 0x1 or 0x0 for true and false. These byte will be base64 encoded.
* ui8 := These byte will be base64 encoded.
* ui8[] := These bytes are getting base64 encoded.
* int[] := the little endian representation of the integers. These bytes are getting base64 encoded.
* bool[] := the bools are getting converted to bytes (0x1 or 0x0 for true and false). These bytes are getting base64 encoded.
* str[] := each string will be URL encoded. Afterwards, join all strings by \n. These bytes are getting base64 encoded.
* f64[] := the little endian representation of the floats. These bytes are getting base64 encoded.
The format of a message is:
command=COMMAND

View File

@ -1,11 +1,16 @@
package ICCC
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"io"
"net/url"
"reflect"
"strconv"
"strings"
)
// Function to convert an ICCC message to HTTP data.
@ -39,74 +44,133 @@ func Message2Data(channel, command string, message interface{}) (data map[string
field := element.Field(i)
keyName := elementType.Field(i).Name
// A buffer for the binary representation:
buffer := new(bytes.Buffer)
// For possible errors:
var errConverter error = nil
// The key for this element:
key := ``
// Look for the right data type:
switch field.Kind().String() {
case `int64`:
key := fmt.Sprintf(`int:%s`, keyName)
data[key] = []string{strconv.FormatInt(field.Int(), 10)}
key = fmt.Sprintf(`int:%s`, keyName)
// Converts the value in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, field.Int())
case `string`:
key := fmt.Sprintf(`str:%s`, keyName)
data[key] = []string{field.String()}
key = fmt.Sprintf(`str:%s`, keyName)
// URL encode the string and copy its bytes to the buffer:
io.Copy(buffer, strings.NewReader(url.QueryEscape(field.String())))
case `float64`:
key := fmt.Sprintf(`f64:%s`, keyName)
data[key] = []string{strconv.FormatFloat(field.Float(), 'f', 9, 64)}
key = fmt.Sprintf(`f64:%s`, keyName)
// Converts the value in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, field.Float())
case `bool`:
key := fmt.Sprintf(`bool:%s`, keyName)
data[key] = []string{strconv.FormatBool(field.Bool())}
key = fmt.Sprintf(`bool:%s`, keyName)
case `uint8`: // byte
key := fmt.Sprintf(`ui8:%s`, keyName)
data[key] = []string{strconv.FormatUint(field.Uint(), 16)}
// Directly convert the bool in a byte:
if field.Bool() {
// Case: True
buffer.WriteByte(0x1) // Write 1
} else {
// Case: False
buffer.WriteByte(0x0) // Write 0
}
case `uint8`: // a byte
key = fmt.Sprintf(`ui8:%s`, keyName)
// uint8 is a byte, thus, write it directly in the buffer:
buffer.WriteByte(byte(field.Uint()))
// Case: Arrays...
case `slice`:
sliceLen := field.Len()
if sliceLen > 0 {
// Which kind of data is this?
sliceKind := field.Index(0).Kind()
key := ``
dataValues := make([]string, sliceLen, sliceLen)
// Select the right data type:
switch sliceKind.String() {
case `uint8`: // bytes
case `uint8`: // a byte array
key = fmt.Sprintf(`ui8[]:%s`, keyName)
values := field.Interface().([]uint8)
for index, value := range values {
dataValues[index] = strconv.FormatUint(uint64(value), 16)
// Directly write the bytes in the buffer:
for _, val := range values {
buffer.WriteByte(byte(val))
}
case `int64`:
key = fmt.Sprintf(`int[]:%s`, keyName)
values := field.Interface().([]int64)
for index, value := range values {
dataValues[index] = strconv.FormatInt(value, 10)
}
// Converts the array in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, values)
case `bool`:
key = fmt.Sprintf(`bool[]:%s`, keyName)
values := field.Interface().([]bool)
for index, value := range values {
dataValues[index] = strconv.FormatBool(value)
// Cannot convert bool to bytes by using binary.Write(). Thus,
// convert it by ower own:
// Loop over all values:
for _, val := range values {
if val {
// If the value is true:
buffer.WriteByte(0x1) // Write 1
} else {
// If the value is false:
buffer.WriteByte(0x0) // Write 0
}
}
case `string`:
key = fmt.Sprintf(`str[]:%s`, keyName)
values := field.Interface().([]string)
for index, value := range values {
dataValues[index] = value
// Mask every string by using a URL encoding.
// This masks e.g. every new-line i.e \n, etc.
// This allows us to combine later the strings by
// using \n:
masked := make([]string, len(values))
// Loop over each string and convert it:
for n, val := range values {
masked[n] = url.QueryEscape(val)
}
// Join all masked strings by using \n and copy the byte array
// representation in the buffer:
io.Copy(buffer, strings.NewReader(strings.Join(masked, "\n")))
case `float64`:
key = fmt.Sprintf(`f64[]:%s`, keyName)
values := field.Interface().([]float64)
for index, value := range values {
dataValues[index] = strconv.FormatFloat(value, 'f', 9, 64)
// Converts the array in a byte array:
errConverter = binary.Write(buffer, binary.LittleEndian, values)
}
}
}
data[key] = dataValues
}
if errConverter != nil {
// An error occurs:
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityUnknown, LM.ImpactUnknown, LM.MessageNamePARSE, `It was not possible to convert an array for an ICCC message.`, fmt.Sprintf("channel='%s'", channel), fmt.Sprintf("command='%s'", command), errConverter.Error())
} else {
// Convert the byte array to a base64 string for the transportation on wire:
data[key] = []string{base64.StdEncoding.EncodeToString(buffer.Bytes())}
}
}