balki/tss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor db out

Browse Source
This commit is contained in:
Balakrishnan Balasubramanian 2022-06-17 17:51:16 -04:00
parent 87990aa922
commit 7e92a042ff
4 changed files with 56 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
app/app.go
View File

@ -6,34 +6,44 @@ import (
"sync"
"time"
"go.balki.me/tss/log"
"github.com/go-logr/logr"
"go.balki.me/tss/db"
"go.balki.me/tss/parser"
"go.balki.me/tss/proxy"
"go.balki.me/tss/schedule"
"go.balki.me/tss/telegram"
)
func Run(configPath string) {
var log = logr.Discard()
func SetLogger(log logr.Logger) {
log = log
}
func Run(configPath string) error {
cfg, err := ParseConfig(configPath)
if err != nil {
log.Panic("failed to parse config", "path", configPath, "err", err)
log.Error(err, "failed to parse config", "path", configPath)
return err
}
scheduler, err := schedule.NewScheduler(cfg.LastSuccessPath)
if err != nil {
log.Panic("failed to create scheduler", "path", cfg.LastSuccessPath, "error", err)
log.Error(err, "failed to create scheduler", "path", cfg.LastSuccessPath)
return err
}
defer func() {
if err := scheduler.Save(); err != nil {
log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err)
log.Error(err, "failed to save last success info", "path", cfg.LastSuccessPath)
}
}()
tgramProxy, err := proxy.GetTransport(cfg.TelegramProxy)
if err != nil {
log.Panic("failed to get proxy transport", "proxyURL", cfg.TelegramProxy, "error", err)
log.Error(err, "failed to get proxy transport", "proxyURL", cfg.TelegramProxy)
return err
}
tgram := telegram.NewTelegramSender(tgramProxy, cfg.TelegramAuthToken)
@ -49,12 +59,13 @@ func Run(configPath string) {
}()
}
wg.Wait()
return nil
}
func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgram telegram.TelegramSender) {
sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
if err != nil {
log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
log.Error(err, "shouldDownload failed", "feed", feed.Name)
return
}
@ -63,9 +74,9 @@ func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgra
return
}
fdb, err := NewDB(dbDir, feed.Name)
fdb, err := db.NewDB(dbDir, feed.Name)
if err != nil {
log.Error("failed to get db", "feed", feed.Name, "db_dir", dbDir, "error", err)
log.Error(err, "failed to get db", "feed", feed.Name, "db_dir", dbDir)
return
}
@ -73,13 +84,13 @@ func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgra
for _, url := range feed.Url {
data, err := Download(url, feed.Proxy)
if err != nil {
log.Error("download failed", "feed", feed.Name, "url", url, "proxy", feed.Proxy, "error", err)
log.Error(err, "download failed", "feed", feed.Name, "url", url, "proxy", feed.Proxy)
return
}
currentEntries, err := parser.ParseFeed(feed.Type, data)
if err != nil {
log.Error("feed parsing failed", "feed", feed.Name, "url", url, "data", data, "error", err)
log.Error(err, "feed parsing failed", "feed", feed.Name, "url", url, "data", data)
return
}
entries = append(entries, currentEntries...)
@ -100,7 +111,7 @@ func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgra
stat.Total = len(entries)
var records []Record
var records []db.Record
var newEntries []parser.FeedEntry
if fdb.IsNewFeed() {
stat.New = stat.Total
@ -111,9 +122,9 @@ func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgra
var filteredEntries []parser.FeedEntry
newEntries, filteredEntries = entries[:ftl], entries[ftl:]
for _, entry := range filteredEntries {
records = append(records, Record{
records = append(records, db.Record{
Time: time.Now(),
Status: Filtered,
Status: db.Filtered,
Filter: "FirstTime",
FeedEntry: entry,
})
@ -123,30 +134,30 @@ func ProcessFeed(feed *FeedCfg, scheduler schedule.Scheduler, dbDir string, tgra
} else {
newEntries, err = fdb.Filter(entries)
if err != nil {
log.Error("failed to filter entries", "feed", feed.Name, "error", err)
log.Error(err, "failed to filter entries", "feed", feed.Name)
return
}
stat.New = len(newEntries)
}
for _, entry := range newEntries {
r := Record{
r := db.Record{
Time: time.Now(),
FeedEntry: entry,
}
err := tgram.SendLink(entry.Link, feed.Channel, feed.Rhash, feed.Title)
if err != nil {
log.Error("failed to send to telegram", "feed", feed.Name, "link", entry.Link, "channel", feed.Channel, "rhash", feed.Rhash, "error", err)
r.Status = Error
log.Error(err, "failed to send to telegram", "feed", feed.Name, "link", entry.Link, "channel", feed.Channel, "rhash", feed.Rhash)
r.Status = db.Error
stat.Error++
} else {
r.Status = Sent
r.Status = db.Sent
}
records = append(records, r)
}
err = fdb.Save(records)
if err != nil {
log.Error("failed to save sent records", "feed", feed.Name, "num_records", len(records), "error", err)
log.Error(err, "failed to save sent records", "feed", feed.Name, "num_records", len(records))
}
}

2
app/db.go → db/db.go
View File

@ -1,4 +1,4 @@
package app
package db
import (
"encoding/csv"

20
log/log.go
View File

@ -39,3 +39,23 @@ func init() {
GLogr = zapr.NewLogger(logger)
}
func GetZapLogger() logr.Logger {
//json logging
cfg := zap.NewProductionConfig()
devCfg := zap.NewDevelopmentConfig()
// Readable time stamp
cfg.EncoderConfig.EncodeTime = devCfg.EncoderConfig.EncodeTime
// Uncomment to enable debug logging
// cfg.Level = devCfg.Level
logger, err := cfg.Build()
if err != nil {
println("unable to intialize zap log")
panic(err)
}
return zapr.NewLogger(logger)
}

6
main.go
View File

@ -9,10 +9,12 @@ import (
)
func main() {
schedule.SetLogger(log.GLogr)
schedule.SetLogger(log.GetZapLogger())
var configPath string
flag.StringVar(&configPath, "config", "./tss.yml", "path to tss.yml config")
flag.Parse()
app.Run(configPath)
if err := app.Run(configPath); err != nil {
panic(err)
}
}