Go/Ocean
Go/Ocean
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented the logging viewer

Browse Source
This commit is contained in:
Thorsten Sommer 2015-06-23 17:22:24 +02:00
parent 3257fe2f32
commit c5633d84f4
10 changed files with 164 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Admin/Templates/LoggingViewer.go
View File

@ -48,7 +48,7 @@ var LoggingViewer string = `
<div class="w-form">
<form class="filterformcontainer" id="wf-form-Filters" name="wf-form-Filters" data-name="Filters" method="post" action="/log">
<input type="hidden" name="LiveView" value="{{.SetLiveView}}">
<input type="hidden" name="CurrentPage" value="{{.CurrentPage}}">
<input type="hidden" name="CurrentPage" value="1">
<div class="w-row">
<div class="w-col w-col-6">
<div class="columns">

11
Log/DeviceDatabase/CacheRefreshMessageNames.go
View File

@ -17,6 +17,17 @@ func cacheRefreshMessageNames() {
for true {
// Read the message names rom the DB:
data := readMessageNamesFromDB()
// Case: The project name was not set now. This happens by the logging system
// after adding this logging device.
if len(data) == 0 {
// Wait for a moment:
time.Sleep(time.Second * 3)
// Try it again:
continue
}
mutexCacheMessageNames.Lock()
// Overwrite the cache:

11
Log/DeviceDatabase/CacheRefreshSenderNames.go
View File

@ -18,6 +18,17 @@ func cacheRefreshSenderNames() {
// Read the sender names from the DB:
data := readSenderNamesFromDB()
// Case: The project name was not set now. This happens by the logging system
// after adding this logging device.
if len(data) == 0 {
// Wait for a moment:
time.Sleep(time.Second * 3)
// Try it again:
continue
}
mutexCacheSenderNames.Lock()
// Overwrite the cache:

152
Log/DeviceDatabase/InitDB.go
View File

@ -105,123 +105,41 @@ func initDatabase() {
//
// Ensure that all necessary indexes are existing:
//
indexProject := mgo.Index{}
indexProject.Key = []string{`Project`}
logDBCollection.EnsureIndex(indexProject)
logDBCollection.EnsureIndexKey(`Sender`)
logDBCollection.EnsureIndexKey(`Category`)
logDBCollection.EnsureIndexKey(`Level`)
logDBCollection.EnsureIndexKey(`Severity`)
logDBCollection.EnsureIndexKey(`Impact`)
logDBCollection.EnsureIndexKey(`MessageName`)
logDBCollection.EnsureIndexKey(`MessageDescription`)
logDBCollection.EnsureIndexKey(`Project`, `Sender`)
logDBCollection.EnsureIndexKey(`Project`, `Category`)
logDBCollection.EnsureIndexKey(`Project`, `Level`)
logDBCollection.EnsureIndexKey(`Project`, `Severity`)
logDBCollection.EnsureIndexKey(`Project`, `Impact`)
logDBCollection.EnsureIndexKey(`Project`, `MessageName`)
logDBCollection.EnsureIndexKey(`Project`, `MessageDescription`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Sender`)
indexSender := mgo.Index{}
indexSender.Key = []string{`Sender`}
logDBCollection.EnsureIndex(indexSender)
// Related to the logging viewer:
logDBCollection.EnsureIndexKey(`Project`) // Logging viewer, case: No filter
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`) // Logging viewer, case: Filter for time
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Sender`, `MessageName`, `Level`, `Category`, `Impact`, `Severity`) // Logging viewer, case: All filters are active
logDBCollection.EnsureIndexKey(`Project`, `Sender`, `MessageName`, `Level`, `Category`, `Impact`, `Severity`) // Logging viewer, case: All filters are active, but no time filter
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Level`, `Category`) // Logging viewer, case: Filter for e.g. app errors from yesterday
logDBCollection.EnsureIndexKey(`Project`, `Level`, `Category`) // Logging viewer, case: Filter for e.g. all app errors
indexCategory := mgo.Index{}
indexCategory.Key = []string{`Category`}
logDBCollection.EnsureIndex(indexCategory)
indexLevel := mgo.Index{}
indexLevel.Key = []string{`Level`}
logDBCollection.EnsureIndex(indexLevel)
indexSeverity := mgo.Index{}
indexSeverity.Key = []string{`Severity`}
logDBCollection.EnsureIndex(indexSeverity)
indexImpact := mgo.Index{}
indexImpact.Key = []string{`Impact`}
logDBCollection.EnsureIndex(indexImpact)
indexMessageName := mgo.Index{}
indexMessageName.Key = []string{`MessageName`}
logDBCollection.EnsureIndex(indexMessageName)
indexMessageDescription := mgo.Index{}
indexMessageDescription.Key = []string{`MessageDescription`}
logDBCollection.EnsureIndex(indexMessageDescription)
indexProjectTimeUTC := mgo.Index{}
indexProjectTimeUTC.Key = []string{`Project`, `TimeUTC`}
logDBCollection.EnsureIndex(indexProjectTimeUTC)
indexProjectSender := mgo.Index{}
indexProjectSender.Key = []string{`Project`, `Sender`}
logDBCollection.EnsureIndex(indexProjectSender)
indexProjectCategory := mgo.Index{}
indexProjectCategory.Key = []string{`Project`, `Category`}
logDBCollection.EnsureIndex(indexProjectCategory)
indexProjectLevel := mgo.Index{}
indexProjectLevel.Key = []string{`Project`, `Level`}
logDBCollection.EnsureIndex(indexProjectLevel)
indexProjectSeverity := mgo.Index{}
indexProjectSeverity.Key = []string{`Project`, `Severity`}
logDBCollection.EnsureIndex(indexProjectSeverity)
indexProjectImpact := mgo.Index{}
indexProjectImpact.Key = []string{`Project`, `Impact`}
logDBCollection.EnsureIndex(indexProjectImpact)
indexProjectMessageName := mgo.Index{}
indexProjectMessageName.Key = []string{`Project`, `MessageName`}
logDBCollection.EnsureIndex(indexProjectMessageName)
indexProjectMessageDescription := mgo.Index{}
indexProjectMessageDescription.Key = []string{`Project`, `MessageDescription`}
logDBCollection.EnsureIndex(indexProjectMessageDescription)
indexProjectTimeUTCSender := mgo.Index{}
indexProjectTimeUTCSender.Key = []string{`Project`, `TimeUTC`, `Sender`}
logDBCollection.EnsureIndex(indexProjectTimeUTCSender)
indexProjectTimeUTCCategory := mgo.Index{}
indexProjectTimeUTCCategory.Key = []string{`Project`, `TimeUTC`, `Category`}
logDBCollection.EnsureIndex(indexProjectTimeUTCCategory)
indexProjectTimeUTCLevel := mgo.Index{}
indexProjectTimeUTCLevel.Key = []string{`Project`, `TimeUTC`, `Level`}
logDBCollection.EnsureIndex(indexProjectTimeUTCLevel)
indexProjectTimeUTCSeverity := mgo.Index{}
indexProjectTimeUTCSeverity.Key = []string{`Project`, `TimeUTC`, `Severity`}
logDBCollection.EnsureIndex(indexProjectTimeUTCSeverity)
indexProjectTimeUTCImpact := mgo.Index{}
indexProjectTimeUTCImpact.Key = []string{`Project`, `TimeUTC`, `Impact`}
logDBCollection.EnsureIndex(indexProjectTimeUTCImpact)
indexProjectTimeUTCMessageName := mgo.Index{}
indexProjectTimeUTCMessageName.Key = []string{`Project`, `TimeUTC`, `MessageName`}
logDBCollection.EnsureIndex(indexProjectTimeUTCMessageName)
indexProjectTimeUTCMessageDescription := mgo.Index{}
indexProjectTimeUTCMessageDescription.Key = []string{`Project`, `TimeUTC`, `MessageDescription`}
logDBCollection.EnsureIndex(indexProjectTimeUTCMessageDescription)
indexTimeUTCSender := mgo.Index{}
indexTimeUTCSender.Key = []string{`TimeUTC`, `Sender`}
logDBCollection.EnsureIndex(indexTimeUTCSender)
indexTimeUTCCategory := mgo.Index{}
indexTimeUTCCategory.Key = []string{`TimeUTC`, `Category`}
logDBCollection.EnsureIndex(indexTimeUTCCategory)
indexTimeUTCLevel := mgo.Index{}
indexTimeUTCLevel.Key = []string{`TimeUTC`, `Level`}
logDBCollection.EnsureIndex(indexTimeUTCLevel)
indexTimeUTCSeverity := mgo.Index{}
indexTimeUTCSeverity.Key = []string{`TimeUTC`, `Severity`}
logDBCollection.EnsureIndex(indexTimeUTCSeverity)
indexTimeUTCImpact := mgo.Index{}
indexTimeUTCImpact.Key = []string{`TimeUTC`, `Impact`}
logDBCollection.EnsureIndex(indexTimeUTCImpact)
indexTimeUTCMessageName := mgo.Index{}
indexTimeUTCMessageName.Key = []string{`TimeUTC`, `MessageName`}
logDBCollection.EnsureIndex(indexTimeUTCMessageName)
indexTimeUTCMessageDescription := mgo.Index{}
indexProjectTimeUTCMessageDescription.Key = []string{`TimeUTC`, `MessageDescription`}
logDBCollection.EnsureIndex(indexTimeUTCMessageDescription)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Category`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Level`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Severity`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `Impact`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `MessageName`)
logDBCollection.EnsureIndexKey(`Project`, `-TimeUTC`, `MessageDescription`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `Sender`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `Category`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `Level`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `Severity`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `Impact`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `MessageName`)
logDBCollection.EnsureIndexKey(`-TimeUTC`, `MessageDescription`)
}

72
Log/DeviceDatabase/ReadCustom.go
View File

@ -5,18 +5,71 @@ import (
LM "github.com/SommerEngineering/Ocean/Log/Meta"
"gopkg.in/mgo.v2/bson"
"math"
"time"
)
func ReadCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMessageName, logSender, logPage string) (events []LogDBEntry, numPages int) {
func ReadCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMessageName, logSender string, logPage int) (events []LogDBEntry, numPages int) {
// The base query:
selection := bson.D{{"Project", projectName}}
//
// TODO => Is currently a stub
// Build the selection statement regarding the admin's choice:
//
// IMPORTANT: The order of the arguments e.g. Project->TimeUTC->Sender...
// is very important to enable the database to use the indexes!
//
// Define the query:
query := logDBCollection.Find(bson.D{}).Sort(`-TimeUTC`) // TODO: projectName!!!
if timeRange != `*` {
nowUTC := time.Now().UTC()
switch timeRange {
case `last5min`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Minute * -5)}}})
case `last30min`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Minute * -30)}}})
case `last60min`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Minute * -60)}}})
case `last24h`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Hour * -24)}}})
case `last7d`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Hour * -24 * 7)}}})
case `lastMonth`:
selection = append(selection, bson.DocElem{"TimeUTC", bson.D{{"$gte", nowUTC.Add(time.Hour * -24 * 31)}}})
}
}
// How many record we have all over?
if logSender != `*` {
selection = append(selection, bson.DocElem{"Sender", logSender})
}
if logMessageName != `*` {
selection = append(selection, bson.DocElem{"MessageName", logMessageName})
}
if logLevel != `*` {
value := `L:` + logLevel
selection = append(selection, bson.DocElem{"Level", value})
}
if logCategory != `*` {
value := `C:` + logCategory
selection = append(selection, bson.DocElem{"Category", value})
}
if logImpact != `*` {
value := `I:` + logImpact
selection = append(selection, bson.DocElem{"Impact", value})
}
if logSeverity != `*` {
value := `S:` + logSeverity
selection = append(selection, bson.DocElem{"Severity", value})
}
// Build the query:
query := logDBCollection.Find(selection)
// How many record we have all over for this project?
numRecords := loggingViewerPageSize
numPages = 1
if number, err := query.Count(); err != nil {
@ -26,8 +79,11 @@ func ReadCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMes
numPages = int(math.Ceil(float64(numRecords) / float64(loggingViewerPageSize)))
}
// Sort all results:
query = query.Sort(`-TimeUTC`)
// Set now the page's record limit:
query = query.Limit(loggingViewerPageSize)
query = query.Skip((logPage - 1) * loggingViewerPageSize).Limit(loggingViewerPageSize)
count := loggingViewerPageSize
// Execute the query and count the results:
@ -45,6 +101,10 @@ func ReadCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMes
// Loop over all entries and store it:
for iter.Next(&entry) {
// Convert the time instance to UTC:
entry.TimeUTC = entry.TimeUTC.UTC()
// Store it:
events[pos] = entry
pos++
}

6
Log/DeviceDatabase/ReadLatest.go
View File

@ -10,7 +10,7 @@ import (
// Read the latest logging events from the database.
func ReadLatest() (events []LogDBEntry, numPages int) {
// Define the query:
query := logDBCollection.Find(bson.D{}).Sort(`-TimeUTC`) // TODO: projectName!!!
query := logDBCollection.Find(bson.D{{"Project", projectName}}).Sort(`-TimeUTC`)
// How many record we have all over?
numRecords := loggingViewerPageSize
@ -41,6 +41,10 @@ func ReadLatest() (events []LogDBEntry, numPages int) {
// Loop over all entries and store it:
for iter.Next(&entry) {
// Convert the time instance to UTC:
entry.TimeUTC = entry.TimeUTC.UTC()
// Store it:
events[pos] = entry
pos++
}

2
Log/DeviceDatabase/ReadMessageNamesFromDB.go
View File

@ -11,7 +11,7 @@ import (
// Read the message names from the database without any cache.
func readMessageNamesFromDB() (result []Scheme.MessageNames) {
var nextMessageNames []string
if err := logDBCollection.Find(bson.D{}).Distinct(`MessageName`, &nextMessageNames); err != nil {
if err := logDBCollection.Find(bson.D{{`Project`, projectName}}).Distinct(`MessageName`, &nextMessageNames); err != nil {
// Case: Error, was not able to write the event to the database:
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.MessageNameDATABASE, `Was not able to read the message names from the database.`, err.Error())
return

2
Log/DeviceDatabase/ReadSenderNamesFromDB.go
View File

@ -11,7 +11,7 @@ import (
// Reads the sender names from the database without any caching.
func readSenderNamesFromDB() (result []Scheme.Sender) {
var nextSenderNames []string
if err := logDBCollection.Find(bson.D{}).Distinct(`Sender`, &nextSenderNames); err != nil {
if err := logDBCollection.Find(bson.D{{"Project", projectName}}).Distinct(`Sender`, &nextSenderNames); err != nil {
// Case: Was not possible to write to the database.
Log.LogShort(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.MessageNameDATABASE, `Was not able to read the sender names from the database.`, err.Error())
return

21
Log/Web/HandlerLog.go
View File

@ -65,13 +65,26 @@ func HandlerWebLog(response http.ResponseWriter, request *http.Request) {
currentPage := request.FormValue(`CurrentPage`)
currentLiveView := request.FormValue(`LiveView`)
// Get the current page as number:
if currentPage != `` {
if number, err := strconv.Atoi(currentPage); err != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityNone, LM.ImpactNone, LM.MessageNameEXECUTE, `Was not able to parse the page number.`, err.Error())
} else {
currentPageNumber = number
}
}
// Store the events for the template:
data.Events, lastPageNumber = readCustom(currentTimeRange, currentLevel, currentCategory, currentImpact, currentSeverity, currentMessageName, currentSender, currentPage)
data.Events, lastPageNumber = readCustom(currentTimeRange, currentLevel, currentCategory, currentImpact, currentSeverity, currentMessageName, currentSender, currentPageNumber)
if strings.ToLower(currentLiveView) == `true` {
data.SetLiveView = true
}
//
// Correct the form's values to '*' for the any-case:
//
if currentLevel != `` {
data.CurrentLevel = currentLevel
} else {
@ -114,11 +127,8 @@ func HandlerWebLog(response http.ResponseWriter, request *http.Request) {
data.CurrentSender = `*`
}
// Calculate the current, last, previous and next page:
if currentPage != `` {
if number, err := strconv.Atoi(currentPage); err != nil {
Log.LogFull(senderName, LM.CategorySYSTEM, LM.LevelERROR, LM.SeverityNone, LM.ImpactNone, LM.MessageNameEXECUTE, `Was not able to parse the page number.`, err.Error())
} else {
currentPageNumber = number
data.CurrentPage = fmt.Sprintf("%d", currentPageNumber)
data.LastPage = fmt.Sprintf("%d", lastPageNumber)
if currentPageNumber+1 > lastPageNumber {
@ -132,7 +142,6 @@ func HandlerWebLog(response http.ResponseWriter, request *http.Request) {
} else {
data.PreviousPage = `1`
}
}
data.CurrentPage = currentPage
} else {
data.CurrentPage = `1`

8
Log/Web/ReadCustom.go
View File

@ -8,7 +8,7 @@ import (
)
// Read a custom event range from the database.
func readCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMessageName, logSender, logPage string) (events []Scheme.LogEvent, numPages int) {
func readCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMessageName, logSender string, logPage int) (events []Scheme.LogEvent, numPages int) {
// Get the custom events:
eventsFromDB, totalNumberPages := DeviceDatabase.ReadCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMessageName, logSender, logPage)
@ -22,7 +22,13 @@ func readCustom(timeRange, logLevel, logCategory, logImpact, logSeverity, logMes
eventFromDB := eventsFromDB[n]
events[n] = Scheme.LogEvent{}
events[n].LogLine = eventFromDB.Format()
// Transfer the log level:
if len(eventFromDB.Level) > 2 {
events[n].LogLevel = fmt.Sprintf("log%s", strings.ToLower(eventFromDB.Level[2:]))
} else {
events[n].LogLevel = fmt.Sprintf("log%s", strings.ToLower(eventFromDB.Level))
}
// Vary the color of each line:
if n%2 == 0 {