109 lines
2.3 KiB
Go
109 lines
2.3 KiB
Go
package app
|
|
|
|
import (
|
|
"io"
|
|
|
|
"go.balki.me/tss/log"
|
|
"go.balki.me/tss/proxy"
|
|
)
|
|
|
|
func Run(configPath string) {
|
|
cfg, err := ParseConfig(configPath)
|
|
|
|
if err != nil {
|
|
log.Panic("failed to parse config", "path", configPath, "err", err)
|
|
}
|
|
|
|
scheduler, err := NewScheduler(cfg.LastSuccessPath)
|
|
|
|
defer func() {
|
|
if err := scheduler.Save(); err != nil {
|
|
log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err)
|
|
}
|
|
}()
|
|
|
|
for _, feed := range cfg.Feeds {
|
|
log.Info("processing feed", "feed", feed.Name)
|
|
ProcessFeed(feed, scheduler, cfg.DbDir)
|
|
}
|
|
|
|
/*
|
|
for _, feed := range cfg.Feeds {
|
|
log.Println("Processing feed", feed.Name)
|
|
data, err := Download(feed.Url, feed.Proxy)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
|
|
for _, feed := range c.Feeds {
|
|
log.Println("Processing feed", feed.Name)
|
|
|
|
links, err := parseFeed(data)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for _, link := range links {
|
|
if alreadySent(link) {
|
|
continue
|
|
}
|
|
err := sendTelegram(link, feed.Channel, feed.Rhash)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
fmt.Println(configPath)
|
|
*/
|
|
}
|
|
|
|
func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string) {
|
|
sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
|
|
if err != nil {
|
|
log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
|
|
return
|
|
}
|
|
|
|
if !sd {
|
|
log.Info("skipping feed due to schedule", "feed", feed.Name)
|
|
return
|
|
}
|
|
|
|
db, err := NewDB(dbDir, feed.Name)
|
|
if err != nil {
|
|
log.Error("failed to get db", "feed", feed.Name, "db_dir", dbDir, "error", err)
|
|
return
|
|
}
|
|
|
|
data, err := Download(feed.Url, feed.Proxy)
|
|
if err != nil {
|
|
log.Error("download failed", "feed", feed.Name, "url", feed.Url, "proxy", feed.Proxy, "error", err)
|
|
return
|
|
}
|
|
|
|
entries, err := ParseFeed(data)
|
|
if err != nil {
|
|
log.Error("feed parsing failed", "feed", feed.Name, "data", data, "error", err)
|
|
return
|
|
}
|
|
|
|
_, err = db.Filter(entries)
|
|
if err != nil {
|
|
log.Error("failed to filter entries", "feed", feed.Name, "error", err)
|
|
}
|
|
}
|
|
|
|
func Download(url string, proxyUrl string) ([]byte, error) {
|
|
client, err := proxy.GetClient(proxyUrl)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res, err := client.Get(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer res.Body.Close()
|
|
return io.ReadAll(res.Body)
|
|
}
|