Add the new NumGen

Added the new and distributed NumGen, which replaces the old one. The
old implementation was based on a master server.
This commit is contained in:
Thorsten Sommer 2015-06-18 17:13:41 +02:00
parent 6dab89a1d4
commit 81c4d15d9f
19 changed files with 342 additions and 300 deletions

View File

@ -42,9 +42,6 @@ func checkConfiguration() {
CheckSingleConfigurationPresentsAndAddIfMissing(`LogStaticFileRequests`, `false`)
CheckSingleConfigurationPresentsAndAddIfMissing(`LogUseDatabaseLogging`, `false`)
CheckSingleConfigurationPresentsAndAddIfMissing(`LogUseConsoleLogging`, `true`)
CheckSingleConfigurationPresentsAndAddIfMissing(`NumGenActiveHosts`, `please replace this with the correct hostname of the host which is the master number generator`)
CheckSingleConfigurationPresentsAndAddIfMissing(`NumGenGetHandler`, `http://localhost:80/next/number`)
CheckSingleConfigurationPresentsAndAddIfMissing(`NumGenBufferSize`, `12`)
CheckSingleConfigurationPresentsAndAddIfMissing(`OceanUtilizeCPUs`, `2`)
CheckSingleConfigurationPresentsAndAddIfMissing(`FilenameWebResources`, `web.zip`)
CheckSingleConfigurationPresentsAndAddIfMissing(`MapStaticFiles2Root`, `false`)

View File

@ -1,6 +0,0 @@
package NumGen
func BadNumber() (result int64) {
result = badNumber64
return
}

71
NumGen/Doc.go Normal file
View File

@ -0,0 +1,71 @@
package NumGen
/*
NumGen is a distributed number generator which generates unique
numbers at a distributed sysem without any centralised or master server.
Range of 64 bit integer: -9223372036854775808 til 9223372036854775807
Construction of the ID:
-----------------------
First comes the time part:
-9223372036854775808 (base number is the smallest possible int64)
+YYYY * 1000000000000000
+ MM * 10000000000000
+ DD * 100000000000
+ HH * 1000000000
+ mm * 10000000
+ ss * 100000
+ fff * 100
Second the machine part:
+ 1000000000000000000 (offset for the machine part to get a fixed length)
+ PID (capped on 822336) * 10000000000000
+ Num CPUs (capped on 99) * 100000000000
+ PageSize (capped on 999999) * 100000
+ Random 0-99 * 1000
+ Sequence 0-999 * 1
Positions:
----------
First part:
9999 maximum year
12 month
31 day
24 hours
59 minutes
59 seconds
999 milliseconds
99991231245959999 Maximum
-9223372036854775808 Base value
Second part:
822336 PID
99 CPUs
999999 Page Size
99 Random
999 Sequence
8223369999999999999 Maximum content value
+1000000000000000000 Offset=Minimum
9223369999999999999 Maximum of part two
The number at the year 9999, if all IDs are used, is:
99989194391184190
Therefore, this is the reserve for a further implementation:
9223372036854775807 - 99989194391184190 = 9123382842463591617
Test & Development:
http://play.golang.org/p/-wbvmFV99D
*/

View File

@ -0,0 +1,69 @@
package NumGen
import (
"math/big"
"math/rand"
"os"
"runtime"
)
// Internal function to generate the second part of the ID.
func generateMachineID() (id int64) {
//
// Please have a look to the main documentation of this package,
// if you are interested about how to calculate the parts.
//
// Using for all calculations big integers to get a precise result!
bigResult := new(big.Int)
bigBase := new(big.Int)
bigNext := new(big.Int)
bigBase.SetString("1000000000000000000", 10)
bigResult = bigBase
pid := int64(os.Getpid())
cpus := int64(runtime.NumCPU())
pageSize := int64(os.Getpagesize())
rnd := int64(rand.Intn(100))
if pid > 822336 {
pid = int64(822336)
}
if cpus > 99 {
cpus = int64(99)
}
if pageSize > 999999 {
pageSize = int64(999999)
}
bigNext = big.NewInt(pid)
bigPID := new(big.Int)
bigPID.SetString("10000000000000", 10)
bigPID = bigPID.Mul(bigPID, bigNext)
bigResult = bigResult.Add(bigResult, bigPID)
bigNext = big.NewInt(cpus)
bigCPU := new(big.Int)
bigCPU.SetString("100000000000", 10)
bigCPU = bigCPU.Mul(bigCPU, bigNext)
bigResult = bigResult.Add(bigResult, bigCPU)
bigNext = big.NewInt(pageSize)
bigPage := new(big.Int)
bigPage.SetString("100000", 10)
bigPage = bigPage.Mul(bigPage, bigNext)
bigResult = bigResult.Add(bigResult, bigPage)
bigNext = big.NewInt(rnd)
bigRND := new(big.Int)
bigRND.SetString("1000", 10)
bigRND = bigRND.Mul(bigRND, bigNext)
bigResult = bigResult.Add(bigResult, bigRND)
id = bigResult.Int64()
return
}

77
NumGen/GenerateTimeID.go Normal file
View File

@ -0,0 +1,77 @@
package NumGen
import (
"math/big"
"time"
)
// Internal function to generate the first part of the ID.
func generateTimeID() (id int64) {
//
// Please have a look to the main documentation of this package,
// if you are interested about how to calculate the parts.
//
// Using for all calculations big integers to get a precise result!
bigResult := new(big.Int)
bigBase := new(big.Int)
bigNext := new(big.Int)
bigBase.SetString("-9223372036854775808", 10)
bigResult = bigBase
t1 := time.Now().UTC()
year := int64(t1.Year())
month := int64(t1.Month())
day := int64(t1.Day())
hours := int64(t1.Hour())
minutes := int64(t1.Minute())
seconds := int64(t1.Second())
milliseconds := int64(float64(t1.Nanosecond()) / 1000000.0)
bigNext = big.NewInt(year)
bigYear := new(big.Int)
bigYear.SetString("1000000000000000", 10)
bigYear = bigYear.Mul(bigYear, bigNext)
bigResult = bigResult.Add(bigResult, bigYear)
bigNext = big.NewInt(month)
bigMonth := new(big.Int)
bigMonth.SetString("10000000000000", 10)
bigMonth = bigMonth.Mul(bigMonth, bigNext)
bigResult = bigResult.Add(bigResult, bigMonth)
bigNext = big.NewInt(day)
bigDay := new(big.Int)
bigDay.SetString("100000000000", 10)
bigDay = bigDay.Mul(bigDay, bigNext)
bigResult = bigResult.Add(bigResult, bigDay)
bigNext = big.NewInt(hours)
bigHours := new(big.Int)
bigHours.SetString("1000000000", 10)
bigHours = bigHours.Mul(bigHours, bigNext)
bigResult = bigResult.Add(bigResult, bigHours)
bigNext = big.NewInt(minutes)
bigMinutes := new(big.Int)
bigMinutes.SetString("10000000", 10)
bigMinutes = bigMinutes.Mul(bigMinutes, bigNext)
bigResult = bigResult.Add(bigResult, bigMinutes)
bigNext = big.NewInt(seconds)
bigSeconds := new(big.Int)
bigSeconds.SetString("100000", 10)
bigSeconds = bigSeconds.Mul(bigSeconds, bigNext)
bigResult = bigResult.Add(bigResult, bigSeconds)
bigNext = big.NewInt(milliseconds)
bigMilliseconds := new(big.Int)
bigMilliseconds.SetString("100", 10)
bigMilliseconds = bigMilliseconds.Mul(bigMilliseconds, bigNext)
bigResult = bigResult.Add(bigResult, bigMilliseconds)
id = bigResult.Int64()
return
}

View File

@ -0,0 +1,39 @@
package NumGen
import (
"math/big"
)
// Internal function to generate a unique ID with the given sequenceNumber.
func generateUniqueID(sequenceNumber int) (id int64) {
// Ensure, that the sequenceNumber is not smaller than 0:
if sequenceNumber < 0 {
sequenceNumber = sequenceNumber * -1
}
// Ensure, that the sequenceNumber is not bigger than 999:
if sequenceNumber > 999 {
sequenceNumber = 999
}
// Convert the sequenceNumber to a 64 bit integer:
seq := int64(sequenceNumber)
// Generate the first and second part of the ID:
timeID := generateTimeID() // First part
machineID := generateMachineID() // Second part
// Convert all numbers to big integers:
bigSEQ := big.NewInt(seq)
bigMachineID := big.NewInt(machineID)
bigTimeID := big.NewInt(timeID)
// Add the parts to get the result:
bigResult := bigTimeID.Add(bigTimeID, bigMachineID)
bigResult = bigResult.Add(bigResult, bigSEQ)
// The result as 64 bit integer:
id = bigResult.Int64()
return
}

View File

@ -1,32 +0,0 @@
package NumGen
import (
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Shutdown"
"net/http"
"net/url"
"strconv"
)
func GetNextInt64(name string) (result int64) {
result = badNumber64
if Shutdown.IsDown() {
return
}
if responseData, errRequest := http.PostForm(getHandler, url.Values{"name": {name}, "password": {correctPassword}}); errRequest != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityCritical, LM.ImpactCritical, LM.MessageNameGENERATOR, `Requesting the next number was not possible.`, errRequest.Error())
return
} else {
nextNumberText := responseData.Header.Get(`nextNumber`)
if number, errAtio := strconv.Atoi(nextNumberText); errAtio != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityCritical, LM.ImpactCritical, LM.MessageNameGENERATOR, `It was not possible to convert the answer into an int64.`, errAtio.Error())
return
} else {
result = int64(number)
return
}
}
}

71
NumGen/GetUniqueID.go Normal file
View File

@ -0,0 +1,71 @@
package NumGen
import (
"time"
)
// Function to generate a unique ID.
func GetUniqueID() (id int64) {
// Exklusive access:
genLock.Lock()
defer genLock.Unlock()
incremented := false
Retry:
// Read the current time
t1 := time.Now().UTC()
t1Milliseconds := int(float64(t1.Nanosecond()) / 1000000.0)
// Calculate the difference to the last generated number:
diff := t1.Sub(genCurrentTime)
// Case 1: Huge difference?
if diff.Seconds() > 1.0 {
genCurrentTime = t1
genCurrentMillisecond = t1Milliseconds
genCurrentMillisecondCounter = 0
id = generateUniqueID(genCurrentMillisecondCounter)
return
}
/*
The fist case is necessary, because the same count of milliseconds
can occur every second, etc. ;-) Therefore, a check only by the
milliseconds is not sufficient.
*/
// Case 2: Small difference?
if t1Milliseconds != genCurrentMillisecond {
genCurrentTime = t1
genCurrentMillisecond = t1Milliseconds
genCurrentMillisecondCounter = 0
id = generateUniqueID(genCurrentMillisecondCounter)
return
}
//
// Case 3: Another number must be generated at the same millisecond!
//
if incremented == false {
genCurrentMillisecondCounter++
incremented = true
}
// Case 3.1: More than 1000 numbers are generated at one millisecond?
if genCurrentMillisecondCounter > 999 {
// This is not possible with the current algorithm!
// Therefore, we force this and all other request to wait some time:
time.Sleep(1 * time.Millisecond)
// Try it again:
goto Retry
// Case 3.2: Less than 1000 numbers are generated at one millisecond?
} else {
// This case is fine:
id = generateUniqueID(genCurrentMillisecondCounter)
return
}
}

View File

@ -1,43 +0,0 @@
package NumGen
import (
"fmt"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Shutdown"
"net/http"
)
func HandlerGetNext(response http.ResponseWriter, request *http.Request) {
if Shutdown.IsDown() {
http.NotFound(response, request)
return
}
if !isActive {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelWARN, LM.SeverityCritical, LM.ImpactNone, LM.MessageNameCONFIGURATION, `Called the get handler on an inactive host.`, `Wrong configuration?`)
http.NotFound(response, request)
return
}
if correctPassword == `` {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityCritical, LM.ImpactCritical, LM.MessageNameSECURITY, `No communication password was set.`)
http.NotFound(response, request)
return
}
name := request.FormValue(`name`)
pwd := request.FormValue(`password`)
if pwd != correctPassword {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelSECURITY, LM.SeverityCritical, LM.ImpactNone, LM.MessageNamePASSWORD, `A wrong password was used to access this system handler.`, `This should never happens: Is this a hacking attempt?`, `IP address of requester=`+request.RemoteAddr)
http.NotFound(response, request)
return
}
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelDEBUG, LM.MessageNameANALYSIS, `Next number requested.`, name, pwd)
channel := requestChannel4Name(name)
nextNumber := <-channel
response.Header().Add(`nextNumber`, fmt.Sprintf(`%d`, nextNumber))
}

View File

@ -1,42 +0,0 @@
package NumGen
import (
"github.com/SommerEngineering/Ocean/ConfigurationDB"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Tools"
"strconv"
"strings"
)
// Init this package.
func init() {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameSTARTUP, `Init the number generator.`)
// Get the exklusive access to the channel list:
channelListLock.Lock()
defer channelListLock.Unlock()
correctPassword = ConfigurationDB.Read(`InternalCommPassword`)
activeHost := ConfigurationDB.Read(`NumGenActiveHosts`)
isActive = strings.Contains(activeHost, Tools.ThisHostname())
getHandler = ConfigurationDB.Read(`NumGenGetHandler`)
if isActive {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelWARN, LM.MessageNameCONFIGURATION, `The number generator is active on this host.`, `This host is producer and consumer.`)
channelBufferSizeText := ConfigurationDB.Read(`NumGenBufferSize`)
if bufferSizeNumber, errBufferSizeNumber := strconv.Atoi(channelBufferSizeText); errBufferSizeNumber != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityCritical, LM.ImpactMiddle, LM.MessageNameCONFIGURATION, `Was not able to parse the configuration value of NumGenBufferSize.`, errBufferSizeNumber.Error(), `Use the default value now!`)
} else {
channelBufferSize = bufferSizeNumber
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameCONFIGURATION, `The buffer size for the number generator was loaded.`, `Buffer size=`+channelBufferSizeText)
}
channelList = make(map[string]chan int64)
initDB()
} else {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelWARN, LM.MessageNameCONFIGURATION, `The number generator is not active on this host.`, `This host is just a consumer.`)
}
}

View File

@ -1,30 +0,0 @@
package NumGen
import (
"github.com/SommerEngineering/Ocean/CustomerDB"
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"gopkg.in/mgo.v2"
)
func initDB() {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameINIT, `Start init of number generator collection.`)
defer Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameINIT, `Done init of number generator collection.`)
// Get the database:
dbSession, db = CustomerDB.DB()
if db == nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityCritical, LM.ImpactCritical, LM.MessageNameDATABASE, `Was not able to get the customer database.`)
return
}
// Get my collection:
collectionNumGen = db.C(`NumGen`)
// Take care about the indexes:
indexName := mgo.Index{}
indexName.Key = []string{`Name`}
indexName.Unique = true
collectionNumGen.EnsureIndex(indexName)
}

View File

@ -1,70 +0,0 @@
package NumGen
import (
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Shutdown"
"gopkg.in/mgo.v2/bson"
"time"
)
func producer(name string) {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameSTARTUP, `The NumGen producer is now starting.`, `name=`+name)
// Get my channel:
myChannel := requestChannel4Name(name)
// Read my next free number:
currentNextFreeNumber := nextFreeNumberFromDatabase(name)
// Where is the next "reload"?
nextReload := currentNextFreeNumber + int64(channelBufferSize)
// Set the next free number to the database:
updateNextFreeNumber2Database(name, nextReload)
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameSTARTUP, `The NumGen producer is now running.`, `name=`+name)
for nextNumber := currentNextFreeNumber; true; {
if Shutdown.IsDown() {
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelINFO, LM.MessageNameSHUTDOWN, `The NumGen producer is now down.`, `name=`+name)
return
}
if nextNumber > nextReload {
nextReload = nextReload + int64(channelBufferSize)
updateNextFreeNumber2Database(name, nextReload)
// Enables the administrator to monitor the frequence of chunks and is able to reconfigure the settings:
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelDEBUG, LM.MessageNamePRODUCER, `The NumGen producer creates the next chunk.`, `name=`+name)
}
// Enqueue the next number:
select {
case myChannel <- nextNumber:
nextNumber++
case <-time.After(time.Millisecond * 500):
}
}
}
func nextFreeNumberFromDatabase(name string) (result int64) {
selection := bson.D{{`Name`, name}}
searchResult := NumberGenScheme{}
count, _ := collectionNumGen.Find(selection).Count()
if count == 1 {
collectionNumGen.Find(selection).One(&searchResult)
result = searchResult.NextFreeNumber
} else {
searchResult.Name = name
searchResult.NextFreeNumber = startValue64
collectionNumGen.Insert(searchResult)
result = searchResult.NextFreeNumber
}
return
}
func updateNextFreeNumber2Database(name string, nextFreeNumber int64) {
selection := bson.D{{`Name`, name}}
collectionNumGen.Update(selection, NumberGenScheme{Name: name, NextFreeNumber: nextFreeNumber})
}

View File

@ -1,40 +0,0 @@
package NumGen
import (
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Shutdown"
)
func requestChannel4Name(name string) (result chan int64) {
if Shutdown.IsDown() {
return
}
if !isActive {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelWARN, LM.SeverityCritical, LM.ImpactNone, LM.MessageNameCONFIGURATION, `Called the requestChannel4Name() on an inactive host.`, `Wrong configuration?`)
return
}
channelListLock.RLock()
channel, isPresent := channelList[name]
channelListLock.RUnlock()
if isPresent {
result = channel
return
}
// Create the entry:
newChannel := make(chan int64, channelBufferSize)
result = newChannel
channelListLock.Lock()
channelList[name] = newChannel
channelListLock.Unlock()
// Create the new producer:
go producer(name)
return
}

View File

@ -1,7 +0,0 @@
package NumGen
// The scheme for the database.
type NumberGenScheme struct {
Name string `bson:"Name"` // A name for this counter.
NextFreeNumber int64 `bson:"NextFreeNumber"` // The next number.
}

View File

@ -5,11 +5,11 @@ import (
LM "github.com/SommerEngineering/Ocean/Log/Meta"
)
// The type for the shutdown function.
type ShutdownFunction struct {
}
// The shutdown handler for this package.
func (a ShutdownFunction) Shutdown() {
Log.LogShort(senderName, LM.CategoryAPP, LM.LevelWARN, LM.MessageNameSHUTDOWN, `Shutting down the number generator.`)
db.Logout()
dbSession.Close()
}

View File

@ -2,26 +2,14 @@ package NumGen
import (
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"gopkg.in/mgo.v2"
"sync"
"time"
)
var (
correctPassword string = ``
// This is the name for logging event from this package:
senderName LM.Sender = `System::NumGen::Producer`
isActive bool = false
getHandler string = ``
db *mgo.Database = nil
dbSession *mgo.Session = nil
collectionNumGen *mgo.Collection = nil
channelBufferSize int = 10
channelList map[string]chan int64 = nil
channelListLock sync.RWMutex = sync.RWMutex{}
)
const (
badNumber64 int64 = 9222222222222222222
startValue64 int64 = -9223372036854775808
senderName LM.Sender = `System::NumGen` // This is the name for logging event from this package
genLock sync.Mutex = sync.Mutex{} // The mutex for the generator
genCurrentTime time.Time = time.Now().UTC() // The time for the last generated number
genCurrentMillisecond int = 0 // The millisecond for the last generated number
genCurrentMillisecondCounter int = 0 // The counter of how many numbers are generated at the same time
)

View File

@ -9,6 +9,7 @@ Ocean is a smart and powerful application framework and server which uses the KI
* A messaging component called ICCC to communicate with all of your servers (or even with other parts at different programming languages at different servers)
* A distributed template engine for web applications
* A distributed half-automated configuration management
* A distributed number generator which produces e.g. customer IDs
* A simple I18N support
* A simple database abstraction which MongoDB as database back-end

View File

@ -8,7 +8,6 @@ import (
"github.com/SommerEngineering/Ocean/Log"
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"github.com/SommerEngineering/Ocean/Log/Web"
"github.com/SommerEngineering/Ocean/NumGen"
"github.com/SommerEngineering/Ocean/Robots"
"github.com/SommerEngineering/Ocean/StaticFiles"
"github.com/SommerEngineering/Ocean/WebContent"
@ -30,9 +29,6 @@ func InitHandlers() {
// Handler for other static files:
Handlers.AddPublicHandler(`/staticFiles/`, StaticFiles.HandlerStaticFiles)
// Handler for the number generator:
Handlers.AddPublicHandler(`/next/number`, NumGen.HandlerGetNext)
// Handler for the robots.txt:
Handlers.AddPublicHandler(`/robots.txt`, Robots.HandlerRobots)
@ -49,9 +45,6 @@ func InitHandlers() {
// Handler for other static files:
Handlers.AddAdminHandler(`/staticFiles/`, StaticFiles.HandlerStaticFiles)
// Handler for the number generator:
Handlers.AddAdminHandler(`/next/number`, NumGen.HandlerGetNext)
// Handler for the ICCC to the private side:
Handlers.AddAdminHandler(`/ICCC`, ICCC.ICCCHandler)

View File

@ -11,6 +11,12 @@ func RandomInteger(max int) (rnd int) {
return
}
// Gets a random 64 bit float between 0.0 and 1.0 (but without 1.0).
func RandomFloat64() (rnd float64) {
rnd = rand.Float64()
return
}
// Gets a random UUID (v4).
func RandomGUID() (guidString string) {
guidString = uuid.NewV4().String()