From 28fe4dcf1cef2baf63efba0d65299758cf3c69eb Mon Sep 17 00:00:00 2001 From: Raf Date: Wed, 13 May 2020 16:39:06 -0400 Subject: [PATCH] Speedup db update for large files --- src/config.go | 6 ++-- src/data.go | 12 +++---- src/screen.go | 4 +-- src/struct-system.go | 3 +- src/xepg.go | 84 +++++++++++++++++++++++++------------------- 5 files changed, 62 insertions(+), 47 deletions(-) diff --git a/src/config.go b/src/config.go index a585bf9..e11d46f 100644 --- a/src/config.go +++ b/src/config.go @@ -46,7 +46,8 @@ func Init() (err error) { System.ServerProtocol.M3U = "http" System.ServerProtocol.WEB = "http" System.ServerProtocol.XML = "http" - System.DVRLimit = 480 + System.PlexChannelLimit = 480 + System.UnfilteredChannelLimit = 480 System.Compatibility = "1.4.4" // FFmpeg Default Einstellungen @@ -229,7 +230,8 @@ func StartSystem(updateProviderFiles bool) (err error) { showInfo(fmt.Sprintf("UUID:%s", Settings.UUID)) showInfo(fmt.Sprintf("Tuner (Plex / Emby):%d", Settings.Tuner)) showInfo(fmt.Sprintf("EPG Source:%s", Settings.EpgSource)) - showInfo(fmt.Sprintf("Plex Channel Limit:%d", System.DVRLimit)) + showInfo(fmt.Sprintf("Plex Channel Limit:%d", System.PlexChannelLimit)) + showInfo(fmt.Sprintf("Unfiltered Channel Limit:%d", System.UnfilteredChannelLimit)) // Providerdaten aktualisieren if len(Settings.Files.M3U) > 0 && Settings.FilesUpdate == true || updateProviderFiles == true { diff --git a/src/data.go b/src/data.go index e88c8a2..8398c9a 100644 --- a/src/data.go +++ b/src/data.go @@ -793,9 +793,9 @@ func buildDatabaseDVR() (err error) { System.ScanInProgress = 1 - Data.Streams.All = make([]interface{}, 0) - Data.Streams.Active = make([]interface{}, 0) - Data.Streams.Inactive = make([]interface{}, 0) + Data.Streams.All = make([]interface{}, 0, System.UnfilteredChannelLimit) + Data.Streams.Active = make([]interface{}, 0, System.UnfilteredChannelLimit) + Data.Streams.Inactive = make([]interface{}, 0, System.UnfilteredChannelLimit) Data.Playlist.M3U.Groups.Text = []string{} Data.Playlist.M3U.Groups.Value = []string{} Data.StreamPreviewUI.Active = []string{} @@ -951,7 +951,7 @@ func buildDatabaseDVR() (err error) { sort.Strings(Data.Playlist.M3U.Groups.Text) sort.Strings(Data.Playlist.M3U.Groups.Value) - if len(Data.Streams.Active) == 0 && len(Data.Streams.All) <= System.DVRLimit && len(Settings.Filter) == 0 { + if len(Data.Streams.Active) == 0 && len(Data.Streams.All) <= System.UnfilteredChannelLimit && len(Settings.Filter) == 0 { Data.Streams.Active = Data.Streams.All Data.Streams.Inactive = make([]interface{}, 0) @@ -960,11 +960,11 @@ func buildDatabaseDVR() (err error) { } - if len(Data.Streams.Active) > System.DVRLimit { + if len(Data.Streams.Active) > System.PlexChannelLimit { showWarning(2000) } - if len(Settings.Filter) == 0 && len(Data.Streams.All) > System.DVRLimit { + if len(Settings.Filter) == 0 && len(Data.Streams.All) > System.UnfilteredChannelLimit { showWarning(2001) } diff --git a/src/screen.go b/src/screen.go index 56876bb..d04cfa8 100644 --- a/src/screen.go +++ b/src/screen.go @@ -300,9 +300,9 @@ func getErrMsg(errCode int) (errMsg string) { // Warnings case 2000: - errMsg = fmt.Sprintf("Plex can not handle more than %d streams. If you do not use Plex, you can ignore this warning.", System.DVRLimit) + errMsg = fmt.Sprintf("Plex can not handle more than %d streams. If you do not use Plex, you can ignore this warning.", System.PlexChannelLimit) case 2001: - errMsg = fmt.Sprintf("%s has loaded more than %d streams. Use the filter to reduce the number of streams.", System.Name, System.DVRLimit) + errMsg = fmt.Sprintf("%s has loaded more than %d streams. Use the filter to reduce the number of streams.", System.Name, System.UnfilteredChannelLimit) case 2002: errMsg = fmt.Sprintf("PMS can not play m3u8 streams") case 2003: diff --git a/src/struct-system.go b/src/struct-system.go index 37f0fcd..b039b36 100644 --- a/src/struct-system.go +++ b/src/struct-system.go @@ -20,7 +20,8 @@ type SystemStruct struct { Dev bool DeviceID string Domain string - DVRLimit int + PlexChannelLimit int + UnfilteredChannelLimit int FFmpeg struct { DefaultOptions string diff --git a/src/xepg.go b/src/xepg.go index 932f373..557166d 100644 --- a/src/xepg.go +++ b/src/xepg.go @@ -253,8 +253,9 @@ func createXEPGMapping() { // XEPG Datenbank erstellen / aktualisieren func createXEPGDatabase() (err error) { - var allChannelNumbers []float64 - Data.Cache.Streams.Active = []string{} + var allChannelNumbers = make([]float64, 0, System.UnfilteredChannelLimit) + Data.Cache.Streams.Active = make([]string, 0, System.UnfilteredChannelLimit) + Data.XEPG.Channels = make(map[string]interface{}, System.UnfilteredChannelLimit) Data.XEPG.Channels, err = loadJSONFileToMap(System.File.XEPG) if err != nil { @@ -281,6 +282,11 @@ func createXEPGDatabase() (err error) { var firstFreeNumber float64 = Settings.MappingFirstChannel + if len(allChannelNumbers) > 0 { + firstFreeNumber = allChannelNumbers[len(allChannelNumbers)-1] //Start with last assigned channel number. Avoids checking from the beginning each time + firstFreeNumber++ + } + newNumber: if indexOfFloat64(firstFreeNumber, allChannelNumbers) == -1 { @@ -296,7 +302,7 @@ func createXEPGDatabase() (err error) { showInfo("XEPG:" + "Update database") - // Kanal mit fehlenden Kanalnummern löschen + // Kanal mit fehlenden Kanalnummern löschen. Delete channel with missing channel numbers for id, dxc := range Data.XEPG.Channels { var xepgChannel XEPGChannelStruct @@ -315,17 +321,24 @@ func createXEPGDatabase() (err error) { } - var xepgChannels = make(map[string]interface{}) + // Make a map of the db channels based on their previously downloaded attributes -- filename, group, title, etc + var xepgChannelsValuesMap = make(map[string]XEPGChannelStruct, System.UnfilteredChannelLimit) + for _, v := range Data.XEPG.Channels { + var channel XEPGChannelStruct + err = json.Unmarshal([]byte(mapToJSON(v)), &channel) + if err != nil { + return + } - for k, v := range Data.XEPG.Channels { - xepgChannels[k] = v + xepgChannelsValuesMap[channel.FileM3UID+channel.Name+channel.GroupTitle+channel.TvgID+channel.TvgName+channel.TvgLogo] = channel } for _, dsa := range Data.Streams.Active { - var channelExists = false // Entscheidet ob ein Kanal neu zu Datenbank hinzugefügt werden soll. - var channelHasUUID = false // Überprüft, ob der Kanal (Stream) eindeutige ID's besitzt - var currentXEPGID string // Aktuelle Datenbank ID (XEPG). Wird verwendet, um den Kanal in der Datenbank mit dem Stream der M3u zu aktualisieren + var channelExists = false // Entscheidet ob ein Kanal neu zu Datenbank hinzugefügt werden soll. Decides whether a channel should be added to the database + var channelHasUUID = false // Überprüft, ob der Kanal (Stream) eindeutige ID's besitzt. Checks whether the channel (stream) has unique IDs + var currentXEPGID string // Aktuelle Datenbank ID (XEPG). Wird verwendet, um den Kanal in der Datenbank mit dem Stream der M3u zu aktualisieren. Current database ID (XEPG) Used to update the channel in the database with the stream of the M3u + var m3uChannel M3UChannelStructXEPG err = json.Unmarshal([]byte(mapToJSON(dsa)), &m3uChannel) @@ -335,42 +348,41 @@ func createXEPGDatabase() (err error) { Data.Cache.Streams.Active = append(Data.Cache.Streams.Active, m3uChannel.Name) - // XEPG Datenbank durchlaufen um nach dem Kanal zu suchen. - for xepg, dxc := range xepgChannels { + // Try to find the channel based on matching all known values. If that fails, then move to full channel scan + if val, ok := xepgChannelsValuesMap[m3uChannel.FileM3UID+m3uChannel.Name+m3uChannel.GroupTitle+m3uChannel.TvgID+m3uChannel.TvgName+m3uChannel.TvgLogo]; ok { + channelExists = true + channelHasUUID = false + currentXEPGID = val.XEPG + } else { - var xepgChannel XEPGChannelStruct - err = json.Unmarshal([]byte(mapToJSON(dxc)), &xepgChannel) - if err != nil { - return - } + // XEPG Datenbank durchlaufen um nach dem Kanal zu suchen. Run through the XEPG database to search for the channel (full scan) + for _, dxc := range xepgChannelsValuesMap { - // Vergleichen des Streams anhand einer UUID in der M3U mit dem Kanal in der Databank - if len(xepgChannel.UUIDValue) > 0 && len(m3uChannel.UUIDValue) > 0 { + // Vergleichen des Streams anhand einer UUID in der M3U mit dem Kanal in der Databank. Compare the stream using a UUID in the M3U with the channel in the database + if len(dxc.UUIDValue) > 0 && len(m3uChannel.UUIDValue) > 0 { - if xepgChannel.UUIDValue == m3uChannel.UUIDValue && xepgChannel.UUIDKey == m3uChannel.UUIDKey { + if dxc.UUIDValue == m3uChannel.UUIDValue && dxc.UUIDKey == m3uChannel.UUIDKey { - channelExists = true - channelHasUUID = true - currentXEPGID = xepg - break + channelExists = true + channelHasUUID = true + currentXEPGID = dxc.XEPG + break + + } + + } else { + // Vergleichen des Streams mit dem Kanal in der Databank anhand des Kanalnamens. Compare the stream to the channel in the database using the channel name + if dxc.Name == m3uChannel.Name { + channelExists = true + currentXEPGID = dxc.XEPG + break + } } - } else { - // Vergleichen des Streams mit dem Kanal in der Databank anhand des Kanalnamens - //fmt.Println(xepgChannel.Name, xepgChannel.UUIDKey, xepgChannel.UUIDValue) - if xepgChannel.Name == m3uChannel.Name { - channelExists = true - currentXEPGID = xepg - break - } - } - } - //os.Exit(0) - switch channelExists { case true: @@ -435,7 +447,7 @@ func createXEPGDatabase() (err error) { } } - + showInfo("XEPG:" + "Save DB file") err = saveMapToJSONFile(System.File.XEPG, Data.XEPG.Channels) if err != nil { return