Bug fix: Image caching (#172)

src/data.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"../src/internal/authentication"
+	"../src/internal/imgcache"
 )
 
 // Einstellungen ändern (WebUI)
@@ -203,30 +204,43 @@ func updateServerSettings(request RequestStruct) (settings SettingsStruct, err e
 
 		if cacheImages == true {
 
-			if Settings.EpgSource == "XEPG" {
+			if Settings.EpgSource == "XEPG" && System.ImageCachingInProgress == 0 {
-
-				go func() {
-
-					if Settings.CacheImages == true {
-
-						createXMLTVFile()
-						cachingImages()
-						createXMLTVFile()
-						createM3UFile()
-
-					} else {
-
-						createXMLTVFile()
-						createM3UFile()
 
+				Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
+				if err != nil {
+					ShowError(err, 0)
 				}
 
+				switch Settings.CacheImages {
+
+				case false:
+					createXMLTVFile()
+					createM3UFile()
+
+				case true:
+					go func() {
+
+						createXMLTVFile()
+						createM3UFile()
+
+						System.ImageCachingInProgress = 1
+						showInfo("Image Caching:Images are cached")
+
+						Data.Cache.Images.Image.Caching()
+						showInfo("Image Caching:Done")
+
+						System.ImageCachingInProgress = 0
+
+						buildXEPG(false)
+
 					}()
 
 				}
 
 			}
 
+		}
+
 		if createXEPGFiles == true {
 
 			go func() {
@@ -496,6 +510,11 @@ func saveXEpgMapping(request RequestStruct) (err error) {
 
 	var tmp = Data.XEPG
 
+	Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
+	if err != nil {
+		ShowError(err, 0)
+	}
+
 	err = json.Unmarshal([]byte(mapToJSON(request.EpgMapping)), &tmp)
 	if err != nil {
 		return
@@ -512,18 +531,8 @@ func saveXEpgMapping(request RequestStruct) (err error) {
 
 		System.ScanInProgress = 1
 		cleanupXEPG()
-		buildXEPG(true)
-
-		go func() {
-
-			createXMLTVFile()
-			createM3UFile()
-			showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
-			go cachingImages()
-
 		System.ScanInProgress = 0
-
+		buildXEPG(true)
-		}()
 
 	} else {
 
@@ -545,15 +554,10 @@ func saveXEpgMapping(request RequestStruct) (err error) {
 			}
 
 			System.ScanInProgress = 1
-
 			cleanupXEPG()
-			buildXEPG(false)
-			createXMLTVFile()
-			createM3UFile()
-			showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
-			go cachingImages()
-
 			System.ScanInProgress = 0
+			buildXEPG(false)
+			showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
 
 			System.BackgroundProcess = false
 

src/images.go

@@ -3,155 +3,15 @@ package src
 import (
   b64 "encoding/base64"
   "fmt"
-  "io"
-  "io/ioutil"
-  "net/http"
-  "net/url"
-  "os"
-  "path/filepath"
   "strings"
 )
 
-func getCacheImageURL(imageURL string) (cacheImageURL string) {
-
-  if Settings.CacheImages == false {
-    return imageURL
-  }
-
-  imageURL = strings.Trim(imageURL, "\r\n")
-
-  p, err := url.Parse(imageURL)
-  if err != nil {
-    // URL konnte nicht geparst werden, die ursprüngliche image url wird zurückgegeben
-    showInfo(fmt.Sprintf("Image Caching:Image URL: %s", imageURL))
-    showWarning(4101)
-    return imageURL
-  }
-  var urlMD5 = getMD5(imageURL)
-  var fileExtension = filepath.Ext(p.Path)
-
-  if len(fileExtension) == 0 {
-    // Keine Dateierweiterung vorhanden, die ursprüngliche image url wird zurückgegeben
-    return imageURL
-  }
-
-  if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesFiles) == -1 {
-    Data.Cache.ImagesFiles = append(Data.Cache.ImagesFiles, urlMD5+fileExtension)
-  }
-
-  if System.ImageCachingInProgress == 1 {
-    return imageURL
-  }
-
-  if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesCache) != -1 {
-
-    cacheImageURL = fmt.Sprintf("%s://%s/images/%s%s", System.ServerProtocol.XML, System.Domain, urlMD5, fileExtension)
-
-  } else {
-
-    if strings.Contains(imageURL, System.Domain+"/images/") == false {
-
-      if indexOfString(imageURL, Data.Cache.ImagesURLS) == -1 {
-        Data.Cache.ImagesURLS = append(Data.Cache.ImagesURLS, imageURL)
-      }
-
-    }
-
-    cacheImageURL = imageURL
-
-  }
-
-  return
-}
-
-func cachingImages() {
-
-  if Settings.CacheImages == false || System.ImageCachingInProgress == 1 {
-    return
-  }
-
-  System.ImageCachingInProgress = 1
-
-  showInfo("Image Caching:Images are cached")
-
-  for _, imageURL := range Data.Cache.ImagesURLS {
-
-    if len(imageURL) > 0 {
-      cacheImage(imageURL)
-    }
-
-  }
-
-  showInfo("Image Caching:Done")
-
-  // Bilder die nicht mehr verwendet werden, werden gelöscht
-  files, err := ioutil.ReadDir(System.Folder.ImagesCache)
-  if err != nil {
-    ShowError(err, 0)
-    return
-  }
-
-  for _, file := range files {
-
-    if indexOfString(file.Name(), Data.Cache.ImagesFiles) == -1 {
-
-      var debug = fmt.Sprintf("Image Caching:Remove file: %s %s %d", System.Folder.ImagesCache+file.Name(), file.Name(), len(file.Name()))
-      showDebug(debug, 1)
-      err := os.RemoveAll(System.Folder.ImagesCache + file.Name())
-      if err != nil {
-        ShowError(err, 0)
-      }
-
-    }
-
-  }
-
-  System.ImageCachingInProgress = 0
-
-  return
-}
-
-func cacheImage(imageURL string) {
-
-  var debug string
-  var urlMD5 = getMD5(imageURL)
-  var fileExtension = filepath.Ext(imageURL)
-
-  debug = fmt.Sprintf("Image Caching:File: %s Download: %s", urlMD5+fileExtension, imageURL)
-  showDebug(debug, 1)
-
-  resp, err := http.Get(imageURL)
-  if err != nil {
-    return
-  }
-  defer resp.Body.Close()
-
-  if resp.StatusCode != http.StatusOK {
-    return
-  }
-
-  var filePath = System.Folder.ImagesCache + urlMD5 + fileExtension
-
-  // Datei speichern
-  file, err := os.Create(filePath)
-  if err != nil {
-    return
-  }
-
-  defer file.Close()
-
-  _, err = io.Copy(file, resp.Body)
-
-  return
-}
-
 func uploadLogo(input, filename string) (logoURL string, err error) {
 
   b64data := input[strings.IndexByte(input, ',')+1:]
 
   // BAse64 in bytes umwandeln un speichern
   sDec, err := b64.StdEncoding.DecodeString(b64data)
-
   if err != nil {
     return
   }

src/internal/imgcache/cache.go

@@ -0,0 +1,178 @@
+package imgcache
+
+import (
+  "fmt"
+  "io"
+  "io/ioutil"
+  "net/http"
+  "net/url"
+  "os"
+  "path/filepath"
+  "strings"
+  "sync"
+)
+
+// Cache : Cache strcut
+type Cache struct {
+  path     string
+  cacheURL string
+  caching  bool
+  images   map[string]string
+  Queue    []string
+  Cache    []string
+  Image    imageFunc
+  sync.RWMutex
+}
+
+type imageFunc struct {
+  GetURL  func(string) string
+  Caching func()
+  Remove  func()
+}
+
+// New : New cahce
+func New(path, chacheURL string, caching bool) (c *Cache, err error) {
+
+  c = &Cache{}
+
+  c.images = make(map[string]string)
+  c.path = path
+  c.cacheURL = chacheURL
+  c.caching = caching
+  c.Queue = []string{}
+  c.Cache = []string{}
+
+  var queue []string
+
+  c.Image.GetURL = func(src string) (cacheURL string) {
+
+    c.Lock()
+    defer c.Unlock()
+
+    src = strings.Trim(src, "\r\n")
+
+    if c.caching == false {
+      return src
+    }
+
+    u, err := url.Parse(src)
+    if err != nil || len(filepath.Ext(u.Path)) == 0 {
+      return src
+    }
+
+    var filename = fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))
+    if cacheURL, ok := c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))]; ok {
+      return cacheURL
+    }
+
+    if indexOfString(filename, c.Cache) == -1 {
+
+      if indexOfString(src, c.Queue) == -1 {
+        c.Queue = append(c.Queue, src)
+      }
+
+    } else {
+      c.images[filename] = c.cacheURL + filename
+      src = c.cacheURL + filename
+    }
+
+    /*
+       if _, err := os.Stat(c.path + filename); err != nil {
+         //c.images[filename] = c.cacheURL + filename
+         if indexOfString(src, c.Queue) == -1 {
+           c.Queue = append(c.Queue, src)
+         }
+       } else {
+         c.images[filename] = c.cacheURL + filename
+       }
+    */
+
+    return src
+  }
+
+  c.Image.Caching = func() {
+
+    c.Lock()
+    defer c.Unlock()
+
+    var filename string
+
+    for _, src := range c.Queue {
+
+      resp, err := http.Get(src)
+      if err != nil {
+        continue
+      }
+      defer resp.Body.Close()
+
+      if resp.StatusCode != http.StatusOK {
+        continue
+      }
+
+      filename = fmt.Sprintf("%s%s%s%s", c.path, string(os.PathSeparator), strToMD5(src), filepath.Ext(src))
+
+      file, err := os.Create(filename)
+      if err != nil {
+        continue
+      }
+
+      defer file.Close()
+
+      _, err = io.Copy(file, resp.Body)
+      if err != nil {
+        continue
+      }
+
+      u, err := url.Parse(src)
+      if err == nil {
+        c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))] = c.cacheURL + filename
+      }
+
+      queue = append(queue, src)
+
+    }
+
+    for _, q := range queue {
+      c.Queue = removeStringFromSlice(q, c.Queue)
+    }
+
+  }
+
+  c.Image.Remove = func() {
+
+    c.Lock()
+    defer c.Unlock()
+
+    files, err := ioutil.ReadDir(c.path)
+    if err != nil {
+      return
+    }
+
+    for _, file := range files {
+
+      switch c.caching {
+
+      case true:
+        if _, ok := c.images[file.Name()]; !ok {
+          os.RemoveAll(c.path + file.Name())
+        }
+
+      case false:
+        os.RemoveAll(c.path + file.Name())
+      }
+
+    }
+
+  }
+
+  files, err := ioutil.ReadDir(c.path)
+  if err != nil {
+    return
+  }
+
+  for _, file := range files {
+    c.Cache = append(c.Cache, file.Name())
+  }
+
+  return
+}

src/internal/imgcache/tools.go

@@ -0,0 +1,34 @@
+package imgcache
+
+import (
+  "crypto/md5"
+  "encoding/hex"
+)
+
+func strToMD5(str string) string {
+  md5Hasher := md5.New()
+  md5Hasher.Write([]byte(str))
+  return hex.EncodeToString(md5Hasher.Sum(nil))
+}
+
+func indexOfString(str string, slice []string) int {
+
+  for i, v := range slice {
+    if str == v {
+      return i
+    }
+  }
+
+  return -1
+}
+
+func removeStringFromSlice(str string, slice []string) []string {
+
+  var i = indexOfString(str, slice)
+
+  if i != -1 {
+    slice = append(slice[:i], slice[i+1:]...)
+  }
+
+  return slice
+}

src/m3u.go

@@ -181,6 +181,7 @@ func checkConditions(streamValues, conditions, coType string) (status bool) {
 // xTeVe M3U Datei erstellen
 func buildM3U(groups []string) (m3u string, err error) {
 
+	var imgc = Data.Cache.Images
 	var m3uChannels = make(map[float64]XEPGChannelStruct)
 	var channelNumbers []float64
 
@@ -223,7 +224,8 @@ func buildM3U(groups []string) (m3u string, err error) {
 	for _, channelNumber := range channelNumbers {
 
 		var channel = m3uChannels[channelNumber]
-		var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, getCacheImageURL(channel.TvgLogo), channel.XGroupTitle, channel.XName)
+
+		var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, imgc.Image.GetURL(channel.TvgLogo), channel.XGroupTitle, channel.XName)
 		var stream, err = createStreamingURL("M3U", channel.FileM3UID, channel.XChannelID, channel.XName, channel.URL)
 		if err == nil {
 			m3u = m3u + parameter + stream + "\n"

src/struct-system.go

@@ -1,5 +1,7 @@
 package src
 
+import "../src/internal/imgcache"
+
 // SystemStruct : Beinhaltet alle Systeminformationen
 type SystemStruct struct {
 	Addresses struct {
@@ -116,6 +118,7 @@ type GitStruct struct {
 // DataStruct : Alle Daten werden hier abgelegt. (Lineup, XMLTV)
 type DataStruct struct {
 	Cache struct {
+		Images      *imgcache.Cache
 		ImagesCache []string
 		ImagesFiles []string
 		ImagesURLS  []string

src/xepg.go

@@ -14,6 +14,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"../src/internal/imgcache"
 )
 
 // Provider XMLTV Datei überprüfen
@@ -44,6 +46,13 @@ func buildXEPG(background bool) {
 
 	System.ScanInProgress = 1
 
+	var err error
+
+	Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
+	if err != nil {
+		ShowError(err, 0)
+	}
+
 	if Settings.EpgSource == "XEPG" {
 
 		switch background {
@@ -58,10 +67,29 @@ func buildXEPG(background bool) {
 				cleanupXEPG()
 				createXMLTVFile()
 				createM3UFile()
-				go cachingImages()
 
 				showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
 
+				if Settings.CacheImages == true && System.ImageCachingInProgress == 0 {
+
+					go func() {
+
+						System.ImageCachingInProgress = 1
+						showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue)))
+
+						Data.Cache.Images.Image.Caching()
+						Data.Cache.Images.Image.Remove()
+						showInfo("Image Caching:Done")
+
+						createXMLTVFile()
+						createM3UFile()
+
+						System.ImageCachingInProgress = 0
+
+					}()
+
+				}
+
 				System.ScanInProgress = 0
 
 				// Cache löschen
@@ -84,7 +112,27 @@ func buildXEPG(background bool) {
 
 				createXMLTVFile()
 				createM3UFile()
-				go cachingImages()
+
+				if Settings.CacheImages == true && System.ImageCachingInProgress == 0 {
+
+					go func() {
+
+						System.ImageCachingInProgress = 1
+						showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue)))
+
+						Data.Cache.Images.Image.Caching()
+						Data.Cache.Images.Image.Remove()
+						showInfo("Image Caching:Done")
+
+						createXMLTVFile()
+						createM3UFile()
+
+						System.ImageCachingInProgress = 0
+
+					}()
+
+				}
+
 				showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
 
 				System.ScanInProgress = 0
@@ -590,6 +638,10 @@ func mapping() (err error) {
 // XMLTV Datei erstellen
 func createXMLTVFile() (err error) {
 
+	// Image Cache
+	// 4edd81ab7c368208cc6448b615051b37.jpg
+	var imgc = Data.Cache.Images
+
 	Data.Cache.ImagesFiles = []string{}
 	Data.Cache.ImagesURLS = []string{}
 	Data.Cache.ImagesCache = []string{}
@@ -637,7 +689,7 @@ func createXMLTVFile() (err error) {
 				// Kanäle
 				var channel Channel
 				channel.ID = xepgChannel.XChannelID
-				channel.Icon = Icon{Src: getCacheImageURL(xepgChannel.TvgLogo)}
+				channel.Icon = Icon{Src: imgc.Image.GetURL(xepgChannel.TvgLogo)}
 				channel.DisplayName = append(channel.DisplayName, DisplayName{Value: xepgChannel.XName})
 
 				xepgXML.Channel = append(xepgXML.Channel, &channel)
@@ -764,6 +816,7 @@ func getProgramData(xepgChannel XEPGChannelStruct) (xepgXML XMLTV, err error) {
 // Dummy Daten erstellen (createXMLTVFile)
 func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) {
 
+	var imgc = Data.Cache.Images
 	var currentTime = time.Now()
 	var dateArray = strings.Fields(currentTime.String())
 	var offset = " " + dateArray[2]
@@ -802,7 +855,7 @@ func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) {
 			}
 
 			if Settings.XepgReplaceMissingImages == true {
-				poster.Src = getCacheImageURL(xepgChannel.TvgLogo)
+				poster.Src = imgc.Image.GetURL(xepgChannel.TvgLogo)
 				epg.Poster = append(epg.Poster, poster)
 			}
 
@@ -849,8 +902,10 @@ func getCategory(program *Program, xmltvProgram *Program, xepgChannel XEPGChanne
 // Programm Poster Cover aus der XMLTV Datei laden
 func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelStruct) {
 
+	var imgc = Data.Cache.Images
+
 	for _, poster := range xmltvProgram.Poster {
-		poster.Src = getCacheImageURL(poster.Src)
+		poster.Src = imgc.Image.GetURL(poster.Src)
 		program.Poster = append(program.Poster, poster)
 	}
 
@@ -858,7 +913,7 @@ func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelS
 
 		if len(xmltvProgram.Poster) == 0 {
 			var poster Poster
-			poster.Src = getCacheImageURL(xepgChannel.TvgLogo)
+			poster.Src = imgc.Image.GetURL(poster.Src)
 			program.Poster = append(program.Poster, poster)
 		}
 

xteve.go

@@ -39,7 +39,7 @@ var GitHub = GitHubStruct{Branch: "master", User: "xteve-project", Repo: "xTeVe-
 const Name = "xTeVe"
 
 // Version : Version, die Build Nummer wird in der main func geparst.
-const Version = "2.1.2.0127"
+const Version = "2.1.2.0128"
 
 // DBVersion : Datanbank Version
 const DBVersion = "2.1.0"