From f43ce0f7c56debb1d26063d9da2b17261c8fde31 Mon Sep 17 00:00:00 2001 From: marmei Date: Sat, 3 Oct 2020 11:56:50 +0200 Subject: [PATCH] Bug fix: Image caching (#172) --- src/data.go | 60 +++++------ src/images.go | 140 -------------------------- src/internal/imgcache/cache.go | 178 +++++++++++++++++++++++++++++++++ src/internal/imgcache/tools.go | 34 +++++++ src/m3u.go | 4 +- src/struct-system.go | 3 + src/xepg.go | 67 +++++++++++-- xteve.go | 2 +- 8 files changed, 312 insertions(+), 176 deletions(-) create mode 100644 src/internal/imgcache/cache.go create mode 100644 src/internal/imgcache/tools.go diff --git a/src/data.go b/src/data.go index de7b21d..0b895f1 100644 --- a/src/data.go +++ b/src/data.go @@ -12,6 +12,7 @@ import ( "time" "../src/internal/authentication" + "../src/internal/imgcache" ) // Einstellungen ändern (WebUI) @@ -203,25 +204,38 @@ func updateServerSettings(request RequestStruct) (settings SettingsStruct, err e if cacheImages == true { - if Settings.EpgSource == "XEPG" { + if Settings.EpgSource == "XEPG" && System.ImageCachingInProgress == 0 { - go func() { + Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages) + if err != nil { + ShowError(err, 0) + } - if Settings.CacheImages == true { + switch Settings.CacheImages { - createXMLTVFile() - cachingImages() - createXMLTVFile() - createM3UFile() + case false: + createXMLTVFile() + createM3UFile() - } else { + case true: + go func() { createXMLTVFile() createM3UFile() - } + System.ImageCachingInProgress = 1 + showInfo("Image Caching:Images are cached") - }() + Data.Cache.Images.Image.Caching() + showInfo("Image Caching:Done") + + System.ImageCachingInProgress = 0 + + buildXEPG(false) + + }() + + } } @@ -496,6 +510,11 @@ func saveXEpgMapping(request RequestStruct) (err error) { var tmp = Data.XEPG + Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages) + if err != nil { + ShowError(err, 0) + } + err = json.Unmarshal([]byte(mapToJSON(request.EpgMapping)), &tmp) if err != nil { return @@ -512,19 +531,9 @@ func saveXEpgMapping(request RequestStruct) (err error) { System.ScanInProgress = 1 cleanupXEPG() + System.ScanInProgress = 0 buildXEPG(true) - go func() { - - createXMLTVFile() - createM3UFile() - showInfo("XEPG:" + fmt.Sprintf("Ready to use")) - go cachingImages() - - System.ScanInProgress = 0 - - }() - } else { // Wenn während des erstellen der Datanbank das Mapping erneut gespeichert wird, wird die Datenbank erst später erneut aktualisiert. @@ -545,15 +554,10 @@ func saveXEpgMapping(request RequestStruct) (err error) { } System.ScanInProgress = 1 - cleanupXEPG() - buildXEPG(false) - createXMLTVFile() - createM3UFile() - showInfo("XEPG:" + fmt.Sprintf("Ready to use")) - go cachingImages() - System.ScanInProgress = 0 + buildXEPG(false) + showInfo("XEPG:" + fmt.Sprintf("Ready to use")) System.BackgroundProcess = false diff --git a/src/images.go b/src/images.go index c6e8551..04dd256 100644 --- a/src/images.go +++ b/src/images.go @@ -3,155 +3,15 @@ package src import ( b64 "encoding/base64" "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" "strings" ) -func getCacheImageURL(imageURL string) (cacheImageURL string) { - - if Settings.CacheImages == false { - return imageURL - } - - imageURL = strings.Trim(imageURL, "\r\n") - - p, err := url.Parse(imageURL) - if err != nil { - // URL konnte nicht geparst werden, die ursprüngliche image url wird zurückgegeben - showInfo(fmt.Sprintf("Image Caching:Image URL: %s", imageURL)) - showWarning(4101) - return imageURL - } - var urlMD5 = getMD5(imageURL) - var fileExtension = filepath.Ext(p.Path) - - if len(fileExtension) == 0 { - // Keine Dateierweiterung vorhanden, die ursprüngliche image url wird zurückgegeben - return imageURL - } - - if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesFiles) == -1 { - Data.Cache.ImagesFiles = append(Data.Cache.ImagesFiles, urlMD5+fileExtension) - } - - if System.ImageCachingInProgress == 1 { - return imageURL - } - - if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesCache) != -1 { - - cacheImageURL = fmt.Sprintf("%s://%s/images/%s%s", System.ServerProtocol.XML, System.Domain, urlMD5, fileExtension) - - } else { - - if strings.Contains(imageURL, System.Domain+"/images/") == false { - - if indexOfString(imageURL, Data.Cache.ImagesURLS) == -1 { - Data.Cache.ImagesURLS = append(Data.Cache.ImagesURLS, imageURL) - } - - } - - cacheImageURL = imageURL - - } - - return -} - -func cachingImages() { - - if Settings.CacheImages == false || System.ImageCachingInProgress == 1 { - return - } - - System.ImageCachingInProgress = 1 - - showInfo("Image Caching:Images are cached") - - for _, imageURL := range Data.Cache.ImagesURLS { - - if len(imageURL) > 0 { - cacheImage(imageURL) - } - - } - - showInfo("Image Caching:Done") - - // Bilder die nicht mehr verwendet werden, werden gelöscht - files, err := ioutil.ReadDir(System.Folder.ImagesCache) - if err != nil { - ShowError(err, 0) - return - } - - for _, file := range files { - - if indexOfString(file.Name(), Data.Cache.ImagesFiles) == -1 { - - var debug = fmt.Sprintf("Image Caching:Remove file: %s %s %d", System.Folder.ImagesCache+file.Name(), file.Name(), len(file.Name())) - showDebug(debug, 1) - err := os.RemoveAll(System.Folder.ImagesCache + file.Name()) - if err != nil { - ShowError(err, 0) - } - - } - - } - - System.ImageCachingInProgress = 0 - - return -} - -func cacheImage(imageURL string) { - - var debug string - var urlMD5 = getMD5(imageURL) - var fileExtension = filepath.Ext(imageURL) - - debug = fmt.Sprintf("Image Caching:File: %s Download: %s", urlMD5+fileExtension, imageURL) - showDebug(debug, 1) - - resp, err := http.Get(imageURL) - if err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return - } - - var filePath = System.Folder.ImagesCache + urlMD5 + fileExtension - - // Datei speichern - file, err := os.Create(filePath) - if err != nil { - return - } - - defer file.Close() - - _, err = io.Copy(file, resp.Body) - - return -} - func uploadLogo(input, filename string) (logoURL string, err error) { b64data := input[strings.IndexByte(input, ',')+1:] // BAse64 in bytes umwandeln un speichern sDec, err := b64.StdEncoding.DecodeString(b64data) - if err != nil { return } diff --git a/src/internal/imgcache/cache.go b/src/internal/imgcache/cache.go new file mode 100644 index 0000000..0337dcc --- /dev/null +++ b/src/internal/imgcache/cache.go @@ -0,0 +1,178 @@ +package imgcache + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" +) + +// Cache : Cache strcut +type Cache struct { + path string + cacheURL string + caching bool + images map[string]string + Queue []string + Cache []string + Image imageFunc + sync.RWMutex +} + +type imageFunc struct { + GetURL func(string) string + Caching func() + Remove func() +} + +// New : New cahce +func New(path, chacheURL string, caching bool) (c *Cache, err error) { + + c = &Cache{} + + c.images = make(map[string]string) + c.path = path + c.cacheURL = chacheURL + c.caching = caching + c.Queue = []string{} + c.Cache = []string{} + + var queue []string + + c.Image.GetURL = func(src string) (cacheURL string) { + + c.Lock() + defer c.Unlock() + + src = strings.Trim(src, "\r\n") + + if c.caching == false { + return src + } + + u, err := url.Parse(src) + if err != nil || len(filepath.Ext(u.Path)) == 0 { + return src + } + + var filename = fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path)) + if cacheURL, ok := c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))]; ok { + return cacheURL + } + + if indexOfString(filename, c.Cache) == -1 { + + if indexOfString(src, c.Queue) == -1 { + c.Queue = append(c.Queue, src) + } + + } else { + c.images[filename] = c.cacheURL + filename + src = c.cacheURL + filename + } + + /* + if _, err := os.Stat(c.path + filename); err != nil { + //c.images[filename] = c.cacheURL + filename + if indexOfString(src, c.Queue) == -1 { + c.Queue = append(c.Queue, src) + } + } else { + c.images[filename] = c.cacheURL + filename + } + */ + + return src + } + + c.Image.Caching = func() { + + c.Lock() + defer c.Unlock() + + var filename string + + for _, src := range c.Queue { + + resp, err := http.Get(src) + if err != nil { + continue + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + continue + } + + filename = fmt.Sprintf("%s%s%s%s", c.path, string(os.PathSeparator), strToMD5(src), filepath.Ext(src)) + + file, err := os.Create(filename) + if err != nil { + continue + } + + defer file.Close() + + _, err = io.Copy(file, resp.Body) + if err != nil { + continue + } + + u, err := url.Parse(src) + if err == nil { + c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))] = c.cacheURL + filename + } + + queue = append(queue, src) + + } + + for _, q := range queue { + c.Queue = removeStringFromSlice(q, c.Queue) + } + + } + + c.Image.Remove = func() { + + c.Lock() + defer c.Unlock() + + files, err := ioutil.ReadDir(c.path) + if err != nil { + return + } + + for _, file := range files { + + switch c.caching { + + case true: + if _, ok := c.images[file.Name()]; !ok { + os.RemoveAll(c.path + file.Name()) + } + + case false: + os.RemoveAll(c.path + file.Name()) + } + + } + + } + + files, err := ioutil.ReadDir(c.path) + if err != nil { + return + } + + for _, file := range files { + c.Cache = append(c.Cache, file.Name()) + } + + return +} diff --git a/src/internal/imgcache/tools.go b/src/internal/imgcache/tools.go new file mode 100644 index 0000000..9d4b622 --- /dev/null +++ b/src/internal/imgcache/tools.go @@ -0,0 +1,34 @@ +package imgcache + +import ( + "crypto/md5" + "encoding/hex" +) + +func strToMD5(str string) string { + md5Hasher := md5.New() + md5Hasher.Write([]byte(str)) + return hex.EncodeToString(md5Hasher.Sum(nil)) +} + +func indexOfString(str string, slice []string) int { + + for i, v := range slice { + if str == v { + return i + } + } + + return -1 +} + +func removeStringFromSlice(str string, slice []string) []string { + + var i = indexOfString(str, slice) + + if i != -1 { + slice = append(slice[:i], slice[i+1:]...) + } + + return slice +} diff --git a/src/m3u.go b/src/m3u.go index d513ac0..30a21c5 100644 --- a/src/m3u.go +++ b/src/m3u.go @@ -181,6 +181,7 @@ func checkConditions(streamValues, conditions, coType string) (status bool) { // xTeVe M3U Datei erstellen func buildM3U(groups []string) (m3u string, err error) { + var imgc = Data.Cache.Images var m3uChannels = make(map[float64]XEPGChannelStruct) var channelNumbers []float64 @@ -223,7 +224,8 @@ func buildM3U(groups []string) (m3u string, err error) { for _, channelNumber := range channelNumbers { var channel = m3uChannels[channelNumber] - var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, getCacheImageURL(channel.TvgLogo), channel.XGroupTitle, channel.XName) + + var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, imgc.Image.GetURL(channel.TvgLogo), channel.XGroupTitle, channel.XName) var stream, err = createStreamingURL("M3U", channel.FileM3UID, channel.XChannelID, channel.XName, channel.URL) if err == nil { m3u = m3u + parameter + stream + "\n" diff --git a/src/struct-system.go b/src/struct-system.go index ec9b58f..d4695cd 100644 --- a/src/struct-system.go +++ b/src/struct-system.go @@ -1,5 +1,7 @@ package src +import "../src/internal/imgcache" + // SystemStruct : Beinhaltet alle Systeminformationen type SystemStruct struct { Addresses struct { @@ -116,6 +118,7 @@ type GitStruct struct { // DataStruct : Alle Daten werden hier abgelegt. (Lineup, XMLTV) type DataStruct struct { Cache struct { + Images *imgcache.Cache ImagesCache []string ImagesFiles []string ImagesURLS []string diff --git a/src/xepg.go b/src/xepg.go index 76f60b7..8a98790 100644 --- a/src/xepg.go +++ b/src/xepg.go @@ -14,6 +14,8 @@ import ( "strconv" "strings" "time" + + "../src/internal/imgcache" ) // Provider XMLTV Datei überprüfen @@ -44,6 +46,13 @@ func buildXEPG(background bool) { System.ScanInProgress = 1 + var err error + + Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages) + if err != nil { + ShowError(err, 0) + } + if Settings.EpgSource == "XEPG" { switch background { @@ -58,10 +67,29 @@ func buildXEPG(background bool) { cleanupXEPG() createXMLTVFile() createM3UFile() - go cachingImages() showInfo("XEPG:" + fmt.Sprintf("Ready to use")) + if Settings.CacheImages == true && System.ImageCachingInProgress == 0 { + + go func() { + + System.ImageCachingInProgress = 1 + showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue))) + + Data.Cache.Images.Image.Caching() + Data.Cache.Images.Image.Remove() + showInfo("Image Caching:Done") + + createXMLTVFile() + createM3UFile() + + System.ImageCachingInProgress = 0 + + }() + + } + System.ScanInProgress = 0 // Cache löschen @@ -84,7 +112,27 @@ func buildXEPG(background bool) { createXMLTVFile() createM3UFile() - go cachingImages() + + if Settings.CacheImages == true && System.ImageCachingInProgress == 0 { + + go func() { + + System.ImageCachingInProgress = 1 + showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue))) + + Data.Cache.Images.Image.Caching() + Data.Cache.Images.Image.Remove() + showInfo("Image Caching:Done") + + createXMLTVFile() + createM3UFile() + + System.ImageCachingInProgress = 0 + + }() + + } + showInfo("XEPG:" + fmt.Sprintf("Ready to use")) System.ScanInProgress = 0 @@ -590,6 +638,10 @@ func mapping() (err error) { // XMLTV Datei erstellen func createXMLTVFile() (err error) { + // Image Cache + // 4edd81ab7c368208cc6448b615051b37.jpg + var imgc = Data.Cache.Images + Data.Cache.ImagesFiles = []string{} Data.Cache.ImagesURLS = []string{} Data.Cache.ImagesCache = []string{} @@ -637,7 +689,7 @@ func createXMLTVFile() (err error) { // Kanäle var channel Channel channel.ID = xepgChannel.XChannelID - channel.Icon = Icon{Src: getCacheImageURL(xepgChannel.TvgLogo)} + channel.Icon = Icon{Src: imgc.Image.GetURL(xepgChannel.TvgLogo)} channel.DisplayName = append(channel.DisplayName, DisplayName{Value: xepgChannel.XName}) xepgXML.Channel = append(xepgXML.Channel, &channel) @@ -764,6 +816,7 @@ func getProgramData(xepgChannel XEPGChannelStruct) (xepgXML XMLTV, err error) { // Dummy Daten erstellen (createXMLTVFile) func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) { + var imgc = Data.Cache.Images var currentTime = time.Now() var dateArray = strings.Fields(currentTime.String()) var offset = " " + dateArray[2] @@ -802,7 +855,7 @@ func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) { } if Settings.XepgReplaceMissingImages == true { - poster.Src = getCacheImageURL(xepgChannel.TvgLogo) + poster.Src = imgc.Image.GetURL(xepgChannel.TvgLogo) epg.Poster = append(epg.Poster, poster) } @@ -849,8 +902,10 @@ func getCategory(program *Program, xmltvProgram *Program, xepgChannel XEPGChanne // Programm Poster Cover aus der XMLTV Datei laden func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelStruct) { + var imgc = Data.Cache.Images + for _, poster := range xmltvProgram.Poster { - poster.Src = getCacheImageURL(poster.Src) + poster.Src = imgc.Image.GetURL(poster.Src) program.Poster = append(program.Poster, poster) } @@ -858,7 +913,7 @@ func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelS if len(xmltvProgram.Poster) == 0 { var poster Poster - poster.Src = getCacheImageURL(xepgChannel.TvgLogo) + poster.Src = imgc.Image.GetURL(poster.Src) program.Poster = append(program.Poster, poster) } diff --git a/xteve.go b/xteve.go index 319727f..c43246e 100644 --- a/xteve.go +++ b/xteve.go @@ -39,7 +39,7 @@ var GitHub = GitHubStruct{Branch: "master", User: "xteve-project", Repo: "xTeVe- const Name = "xTeVe" // Version : Version, die Build Nummer wird in der main func geparst. -const Version = "2.1.2.0127" +const Version = "2.1.2.0128" // DBVersion : Datanbank Version const DBVersion = "2.1.0"