nathan/xTeVe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug fix: Image caching (#172)

Browse Source
This commit is contained in:
marmei
2020-10-03 11:56:50 +02:00
parent db59f7ef37
commit f43ce0f7c5
8 changed files with 312 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
src/data.go
View File

@@ -12,6 +12,7 @@ import (
"time"
"../src/internal/authentication"
"../src/internal/imgcache"
)
// Einstellungen ändern (WebUI)
@@ -203,30 +204,43 @@ func updateServerSettings(request RequestStruct) (settings SettingsStruct, err e
if cacheImages == true {
if Settings.EpgSource == "XEPG" {
go func() {
if Settings.CacheImages == true {
createXMLTVFile()
cachingImages()
createXMLTVFile()
createM3UFile()
} else {
createXMLTVFile()
createM3UFile()
if Settings.EpgSource == "XEPG" && System.ImageCachingInProgress == 0 {
Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
if err != nil {
ShowError(err, 0)
}
switch Settings.CacheImages {
case false:
createXMLTVFile()
createM3UFile()
case true:
go func() {
createXMLTVFile()
createM3UFile()
System.ImageCachingInProgress = 1
showInfo("Image Caching:Images are cached")
Data.Cache.Images.Image.Caching()
showInfo("Image Caching:Done")
System.ImageCachingInProgress = 0
buildXEPG(false)
}()
}
}
}
if createXEPGFiles == true {
go func() {
@@ -496,6 +510,11 @@ func saveXEpgMapping(request RequestStruct) (err error) {
var tmp = Data.XEPG
Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
if err != nil {
ShowError(err, 0)
}
err = json.Unmarshal([]byte(mapToJSON(request.EpgMapping)), &tmp)
if err != nil {
return
@@ -512,18 +531,8 @@ func saveXEpgMapping(request RequestStruct) (err error) {
System.ScanInProgress = 1
cleanupXEPG()
buildXEPG(true)
go func() {
createXMLTVFile()
createM3UFile()
showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
go cachingImages()
System.ScanInProgress = 0
}()
buildXEPG(true)
} else {
@@ -545,15 +554,10 @@ func saveXEpgMapping(request RequestStruct) (err error) {
}
System.ScanInProgress = 1
cleanupXEPG()
buildXEPG(false)
createXMLTVFile()
createM3UFile()
showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
go cachingImages()
System.ScanInProgress = 0
buildXEPG(false)
showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
System.BackgroundProcess = false

140
src/images.go
View File

@@ -3,155 +3,15 @@ package src
import (
b64 "encoding/base64"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
)
func getCacheImageURL(imageURL string) (cacheImageURL string) {
if Settings.CacheImages == false {
return imageURL
}
imageURL = strings.Trim(imageURL, "\r\n")
p, err := url.Parse(imageURL)
if err != nil {
// URL konnte nicht geparst werden, die ursprüngliche image url wird zurückgegeben
showInfo(fmt.Sprintf("Image Caching:Image URL: %s", imageURL))
showWarning(4101)
return imageURL
}
var urlMD5 = getMD5(imageURL)
var fileExtension = filepath.Ext(p.Path)
if len(fileExtension) == 0 {
// Keine Dateierweiterung vorhanden, die ursprüngliche image url wird zurückgegeben
return imageURL
}
if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesFiles) == -1 {
Data.Cache.ImagesFiles = append(Data.Cache.ImagesFiles, urlMD5+fileExtension)
}
if System.ImageCachingInProgress == 1 {
return imageURL
}
if indexOfString(urlMD5+fileExtension, Data.Cache.ImagesCache) != -1 {
cacheImageURL = fmt.Sprintf("%s://%s/images/%s%s", System.ServerProtocol.XML, System.Domain, urlMD5, fileExtension)
} else {
if strings.Contains(imageURL, System.Domain+"/images/") == false {
if indexOfString(imageURL, Data.Cache.ImagesURLS) == -1 {
Data.Cache.ImagesURLS = append(Data.Cache.ImagesURLS, imageURL)
}
}
cacheImageURL = imageURL
}
return
}
func cachingImages() {
if Settings.CacheImages == false || System.ImageCachingInProgress == 1 {
return
}
System.ImageCachingInProgress = 1
showInfo("Image Caching:Images are cached")
for _, imageURL := range Data.Cache.ImagesURLS {
if len(imageURL) > 0 {
cacheImage(imageURL)
}
}
showInfo("Image Caching:Done")
// Bilder die nicht mehr verwendet werden, werden gelöscht
files, err := ioutil.ReadDir(System.Folder.ImagesCache)
if err != nil {
ShowError(err, 0)
return
}
for _, file := range files {
if indexOfString(file.Name(), Data.Cache.ImagesFiles) == -1 {
var debug = fmt.Sprintf("Image Caching:Remove file: %s %s %d", System.Folder.ImagesCache+file.Name(), file.Name(), len(file.Name()))
showDebug(debug, 1)
err := os.RemoveAll(System.Folder.ImagesCache + file.Name())
if err != nil {
ShowError(err, 0)
}
}
}
System.ImageCachingInProgress = 0
return
}
func cacheImage(imageURL string) {
var debug string
var urlMD5 = getMD5(imageURL)
var fileExtension = filepath.Ext(imageURL)
debug = fmt.Sprintf("Image Caching:File: %s Download: %s", urlMD5+fileExtension, imageURL)
showDebug(debug, 1)
resp, err := http.Get(imageURL)
if err != nil {
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return
}
var filePath = System.Folder.ImagesCache + urlMD5 + fileExtension
// Datei speichern
file, err := os.Create(filePath)
if err != nil {
return
}
defer file.Close()
_, err = io.Copy(file, resp.Body)
return
}
func uploadLogo(input, filename string) (logoURL string, err error) {
b64data := input[strings.IndexByte(input, ',')+1:]
// BAse64 in bytes umwandeln un speichern
sDec, err := b64.StdEncoding.DecodeString(b64data)
if err != nil {
return
}

178
src/internal/imgcache/cache.go Normal file
View File

@@ -0,0 +1,178 @@
package imgcache
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
)
// Cache : Cache strcut
type Cache struct {
path string
cacheURL string
caching bool
images map[string]string
Queue []string
Cache []string
Image imageFunc
sync.RWMutex
}
type imageFunc struct {
GetURL func(string) string
Caching func()
Remove func()
}
// New : New cahce
func New(path, chacheURL string, caching bool) (c *Cache, err error) {
c = &Cache{}
c.images = make(map[string]string)
c.path = path
c.cacheURL = chacheURL
c.caching = caching
c.Queue = []string{}
c.Cache = []string{}
var queue []string
c.Image.GetURL = func(src string) (cacheURL string) {
c.Lock()
defer c.Unlock()
src = strings.Trim(src, "\r\n")
if c.caching == false {
return src
}
u, err := url.Parse(src)
if err != nil || len(filepath.Ext(u.Path)) == 0 {
return src
}
var filename = fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))
if cacheURL, ok := c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))]; ok {
return cacheURL
}
if indexOfString(filename, c.Cache) == -1 {
if indexOfString(src, c.Queue) == -1 {
c.Queue = append(c.Queue, src)
}
} else {
c.images[filename] = c.cacheURL + filename
src = c.cacheURL + filename
}
/*
if _, err := os.Stat(c.path + filename); err != nil {
//c.images[filename] = c.cacheURL + filename
if indexOfString(src, c.Queue) == -1 {
c.Queue = append(c.Queue, src)
}
} else {
c.images[filename] = c.cacheURL + filename
}
*/
return src
}
c.Image.Caching = func() {
c.Lock()
defer c.Unlock()
var filename string
for _, src := range c.Queue {
resp, err := http.Get(src)
if err != nil {
continue
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
continue
}
filename = fmt.Sprintf("%s%s%s%s", c.path, string(os.PathSeparator), strToMD5(src), filepath.Ext(src))
file, err := os.Create(filename)
if err != nil {
continue
}
defer file.Close()
_, err = io.Copy(file, resp.Body)
if err != nil {
continue
}
u, err := url.Parse(src)
if err == nil {
c.images[fmt.Sprintf("%s%s", strToMD5(src), filepath.Ext(u.Path))] = c.cacheURL + filename
}
queue = append(queue, src)
}
for _, q := range queue {
c.Queue = removeStringFromSlice(q, c.Queue)
}
}
c.Image.Remove = func() {
c.Lock()
defer c.Unlock()
files, err := ioutil.ReadDir(c.path)
if err != nil {
return
}
for _, file := range files {
switch c.caching {
case true:
if _, ok := c.images[file.Name()]; !ok {
os.RemoveAll(c.path + file.Name())
}
case false:
os.RemoveAll(c.path + file.Name())
}
}
}
files, err := ioutil.ReadDir(c.path)
if err != nil {
return
}
for _, file := range files {
c.Cache = append(c.Cache, file.Name())
}
return
}

34
src/internal/imgcache/tools.go Normal file
View File

@@ -0,0 +1,34 @@
package imgcache
import (
"crypto/md5"
"encoding/hex"
)
func strToMD5(str string) string {
md5Hasher := md5.New()
md5Hasher.Write([]byte(str))
return hex.EncodeToString(md5Hasher.Sum(nil))
}
func indexOfString(str string, slice []string) int {
for i, v := range slice {
if str == v {
return i
}
}
return -1
}
func removeStringFromSlice(str string, slice []string) []string {
var i = indexOfString(str, slice)
if i != -1 {
slice = append(slice[:i], slice[i+1:]...)
}
return slice
}

4
src/m3u.go
View File

@@ -181,6 +181,7 @@ func checkConditions(streamValues, conditions, coType string) (status bool) {
// xTeVe M3U Datei erstellen
func buildM3U(groups []string) (m3u string, err error) {
var imgc = Data.Cache.Images
var m3uChannels = make(map[float64]XEPGChannelStruct)
var channelNumbers []float64
@@ -223,7 +224,8 @@ func buildM3U(groups []string) (m3u string, err error) {
for _, channelNumber := range channelNumbers {
var channel = m3uChannels[channelNumber]
var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, getCacheImageURL(channel.TvgLogo), channel.XGroupTitle, channel.XName)
var parameter = fmt.Sprintf(`#EXTINF:0 channelID="%s" tvg-chno="%s" tvg-name="%s" tvg-id="%s" tvg-logo="%s" group-title="%s",%s`+"\n", channel.XEPG, channel.XChannelID, channel.XName, channel.XChannelID, imgc.Image.GetURL(channel.TvgLogo), channel.XGroupTitle, channel.XName)
var stream, err = createStreamingURL("M3U", channel.FileM3UID, channel.XChannelID, channel.XName, channel.URL)
if err == nil {
m3u = m3u + parameter + stream + "\n"

3
src/struct-system.go
View File

@@ -1,5 +1,7 @@
package src
import "../src/internal/imgcache"
// SystemStruct : Beinhaltet alle Systeminformationen
type SystemStruct struct {
Addresses struct {
@@ -116,6 +118,7 @@ type GitStruct struct {
// DataStruct : Alle Daten werden hier abgelegt. (Lineup, XMLTV)
type DataStruct struct {
Cache struct {
Images *imgcache.Cache
ImagesCache []string
ImagesFiles []string
ImagesURLS []string

67
src/xepg.go
View File

@@ -14,6 +14,8 @@ import (
"strconv"
"strings"
"time"
"../src/internal/imgcache"
)
// Provider XMLTV Datei überprüfen
@@ -44,6 +46,13 @@ func buildXEPG(background bool) {
System.ScanInProgress = 1
var err error
Data.Cache.Images, err = imgcache.New(System.Folder.ImagesCache, fmt.Sprintf("%s://%s/images/", System.ServerProtocol.WEB, System.Domain), Settings.CacheImages)
if err != nil {
ShowError(err, 0)
}
if Settings.EpgSource == "XEPG" {
switch background {
@@ -58,10 +67,29 @@ func buildXEPG(background bool) {
cleanupXEPG()
createXMLTVFile()
createM3UFile()
go cachingImages()
showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
if Settings.CacheImages == true && System.ImageCachingInProgress == 0 {
go func() {
System.ImageCachingInProgress = 1
showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue)))
Data.Cache.Images.Image.Caching()
Data.Cache.Images.Image.Remove()
showInfo("Image Caching:Done")
createXMLTVFile()
createM3UFile()
System.ImageCachingInProgress = 0
}()
}
System.ScanInProgress = 0
// Cache löschen
@@ -84,7 +112,27 @@ func buildXEPG(background bool) {
createXMLTVFile()
createM3UFile()
go cachingImages()
if Settings.CacheImages == true && System.ImageCachingInProgress == 0 {
go func() {
System.ImageCachingInProgress = 1
showInfo(fmt.Sprintf("Image Caching:Images are cached (%d)", len(Data.Cache.Images.Queue)))
Data.Cache.Images.Image.Caching()
Data.Cache.Images.Image.Remove()
showInfo("Image Caching:Done")
createXMLTVFile()
createM3UFile()
System.ImageCachingInProgress = 0
}()
}
showInfo("XEPG:" + fmt.Sprintf("Ready to use"))
System.ScanInProgress = 0
@@ -590,6 +638,10 @@ func mapping() (err error) {
// XMLTV Datei erstellen
func createXMLTVFile() (err error) {
// Image Cache
// 4edd81ab7c368208cc6448b615051b37.jpg
var imgc = Data.Cache.Images
Data.Cache.ImagesFiles = []string{}
Data.Cache.ImagesURLS = []string{}
Data.Cache.ImagesCache = []string{}
@@ -637,7 +689,7 @@ func createXMLTVFile() (err error) {
// Kanäle
var channel Channel
channel.ID = xepgChannel.XChannelID
channel.Icon = Icon{Src: getCacheImageURL(xepgChannel.TvgLogo)}
channel.Icon = Icon{Src: imgc.Image.GetURL(xepgChannel.TvgLogo)}
channel.DisplayName = append(channel.DisplayName, DisplayName{Value: xepgChannel.XName})
xepgXML.Channel = append(xepgXML.Channel, &channel)
@@ -764,6 +816,7 @@ func getProgramData(xepgChannel XEPGChannelStruct) (xepgXML XMLTV, err error) {
// Dummy Daten erstellen (createXMLTVFile)
func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) {
var imgc = Data.Cache.Images
var currentTime = time.Now()
var dateArray = strings.Fields(currentTime.String())
var offset = " " + dateArray[2]
@@ -802,7 +855,7 @@ func createDummyProgram(xepgChannel XEPGChannelStruct) (dummyXMLTV XMLTV) {
}
if Settings.XepgReplaceMissingImages == true {
poster.Src = getCacheImageURL(xepgChannel.TvgLogo)
poster.Src = imgc.Image.GetURL(xepgChannel.TvgLogo)
epg.Poster = append(epg.Poster, poster)
}
@@ -849,8 +902,10 @@ func getCategory(program *Program, xmltvProgram *Program, xepgChannel XEPGChanne
// Programm Poster Cover aus der XMLTV Datei laden
func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelStruct) {
var imgc = Data.Cache.Images
for _, poster := range xmltvProgram.Poster {
poster.Src = getCacheImageURL(poster.Src)
poster.Src = imgc.Image.GetURL(poster.Src)
program.Poster = append(program.Poster, poster)
}
@@ -858,7 +913,7 @@ func getPoster(program *Program, xmltvProgram *Program, xepgChannel XEPGChannelS
if len(xmltvProgram.Poster) == 0 {
var poster Poster
poster.Src = getCacheImageURL(xepgChannel.TvgLogo)
poster.Src = imgc.Image.GetURL(poster.Src)
program.Poster = append(program.Poster, poster)
}

2
xteve.go
View File

@@ -39,7 +39,7 @@ var GitHub = GitHubStruct{Branch: "master", User: "xteve-project", Repo: "xTeVe-
const Name = "xTeVe"
// Version : Version, die Build Nummer wird in der main func geparst.
const Version = "2.1.2.0127"
const Version = "2.1.2.0128"
// DBVersion : Datanbank Version
const DBVersion = "2.1.0"