Almost working. csv last record is not written properly

This commit is contained in:
Balakrishnan Balasubramanian 2022-05-02 00:36:27 -04:00
parent f5e4628459
commit 7a8e5029e8
5 changed files with 26 additions and 11 deletions

View File

@ -71,6 +71,8 @@ func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string, tgram telegra
return return
} }
scheduler.Good(feed.Name)
filteredEntries, err := db.Filter(entries) filteredEntries, err := db.Filter(entries)
if err != nil { if err != nil {
log.Error("failed to filter entries", "feed", feed.Name, "error", err) log.Error("failed to filter entries", "feed", feed.Name, "error", err)

View File

@ -2,10 +2,14 @@ package app
import ( import (
"encoding/csv" "encoding/csv"
"errors"
"fmt" "fmt"
"os" "os"
"path" "path"
"strings"
"time" "time"
"go.balki.me/tss/log"
) )
type Status string type Status string
@ -37,8 +41,14 @@ type db struct {
func NewDB(storageDir, feedName string) (DB, error) { func NewDB(storageDir, feedName string) (DB, error) {
dbPath := path.Join(storageDir, fmt.Sprintf("%s.csv", feedName)) dbPath := path.Join(storageDir, fmt.Sprintf("%s.csv", feedName))
db := db{dbPath: dbPath}
db.seenLinks = map[string]struct{}{}
f, err := os.Open(dbPath) f, err := os.Open(dbPath)
if err != nil { if err != nil {
if errors.Is(err, os.ErrNotExist) {
log.Info("db file does not exist, will be created", "feed", feedName, "path", dbPath)
return &db, nil
}
return nil, err return nil, err
} }
defer f.Close() defer f.Close()
@ -47,8 +57,6 @@ func NewDB(storageDir, feedName string) (DB, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to parse csv, path:%v, error:%w", dbPath, err) return nil, fmt.Errorf("failed to parse csv, path:%v, error:%w", dbPath, err)
} }
db := db{dbPath: dbPath}
db.seenLinks = map[string]struct{}{}
for _, rec := range records { for _, rec := range records {
var recStatus Status = Status(rec[2]) var recStatus Status = Status(rec[2])
if recStatus == Sent || recStatus == Filtered { if recStatus == Sent || recStatus == Filtered {
@ -73,7 +81,10 @@ func (d *db) Save(records []Record) error {
if err != nil { if err != nil {
return err return err
} }
defer f.Close() defer func() {
f.Sync()
f.Close()
}()
csvw := csv.NewWriter(f) csvw := csv.NewWriter(f)
if len(d.seenLinks) == 0 { //New file, write header if len(d.seenLinks) == 0 { //New file, write header
csvw.Write([]string{ csvw.Write([]string{
@ -90,7 +101,7 @@ func (d *db) Save(records []Record) error {
r.FeedEntry.Link, r.FeedEntry.Link,
string(r.Status), string(r.Status),
"-", "-",
r.FeedEntry.Content, strings.ReplaceAll(r.FeedEntry.Content, "\n", " "),
}) })
} }
return nil return nil

View File

@ -3,11 +3,11 @@ package app
import ( import (
"errors" "errors"
"fmt" "fmt"
"log"
"os" "os"
"time" "time"
"github.com/robfig/cron/v3" "github.com/robfig/cron/v3"
"go.balki.me/tss/log"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@ -28,7 +28,7 @@ func NewScheduler(filePath string) (*Scheduler, error) {
return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err) return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
} }
f.Close() f.Close()
log.Printf("file: %v does not exist. Will be created\n", filePath) log.Info("scheduler file does not exist, will be created", "path", filePath)
} else { } else {
err = yaml.Unmarshal(data, &s.lastSuccessTime) err = yaml.Unmarshal(data, &s.lastSuccessTime)
if err != nil { if err != nil {

View File

@ -26,13 +26,14 @@ type telegramSender struct {
} }
func (ts *telegramSender) SendLink(link string, channel string, rhash string) error { func (ts *telegramSender) SendLink(link string, channel string, rhash string) error {
msg := struct { msg := struct {
ChatID string `json:"chat_id"` ChatID string `json:"chat_id"`
Text string `json:"text"` Text string `json:"text"`
ParseMode string `json:"parse_mode"`
}{ }{
ChatID: channel, ChatID: channel,
Text: fmt.Sprintf(`<a href="%s">➢</a> <a href="%s">Link</a>`, genIVLink(link, rhash), link), Text: fmt.Sprintf(`<a href="%s">➢</a> <a href="%s">Link</a>`, genIVLink(link, rhash), link),
ParseMode: "HTML",
} }
data, err := json.Marshal(msg) data, err := json.Marshal(msg)

1
tss.yml Symbolic link
View File

@ -0,0 +1 @@
w.tmp/tss.yaml