Almost working. csv last record is not written properly
This commit is contained in:
parent
f5e4628459
commit
7a8e5029e8
@ -71,6 +71,8 @@ func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string, tgram telegra
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
scheduler.Good(feed.Name)
|
||||||
|
|
||||||
filteredEntries, err := db.Filter(entries)
|
filteredEntries, err := db.Filter(entries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("failed to filter entries", "feed", feed.Name, "error", err)
|
log.Error("failed to filter entries", "feed", feed.Name, "error", err)
|
||||||
|
19
app/db.go
19
app/db.go
@ -2,10 +2,14 @@ package app
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"go.balki.me/tss/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Status string
|
type Status string
|
||||||
@ -37,8 +41,14 @@ type db struct {
|
|||||||
|
|
||||||
func NewDB(storageDir, feedName string) (DB, error) {
|
func NewDB(storageDir, feedName string) (DB, error) {
|
||||||
dbPath := path.Join(storageDir, fmt.Sprintf("%s.csv", feedName))
|
dbPath := path.Join(storageDir, fmt.Sprintf("%s.csv", feedName))
|
||||||
|
db := db{dbPath: dbPath}
|
||||||
|
db.seenLinks = map[string]struct{}{}
|
||||||
f, err := os.Open(dbPath)
|
f, err := os.Open(dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
log.Info("db file does not exist, will be created", "feed", feedName, "path", dbPath)
|
||||||
|
return &db, nil
|
||||||
|
}
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
@ -47,8 +57,6 @@ func NewDB(storageDir, feedName string) (DB, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to parse csv, path:%v, error:%w", dbPath, err)
|
return nil, fmt.Errorf("failed to parse csv, path:%v, error:%w", dbPath, err)
|
||||||
}
|
}
|
||||||
db := db{dbPath: dbPath}
|
|
||||||
db.seenLinks = map[string]struct{}{}
|
|
||||||
for _, rec := range records {
|
for _, rec := range records {
|
||||||
var recStatus Status = Status(rec[2])
|
var recStatus Status = Status(rec[2])
|
||||||
if recStatus == Sent || recStatus == Filtered {
|
if recStatus == Sent || recStatus == Filtered {
|
||||||
@ -73,7 +81,10 @@ func (d *db) Save(records []Record) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer func() {
|
||||||
|
f.Sync()
|
||||||
|
f.Close()
|
||||||
|
}()
|
||||||
csvw := csv.NewWriter(f)
|
csvw := csv.NewWriter(f)
|
||||||
if len(d.seenLinks) == 0 { //New file, write header
|
if len(d.seenLinks) == 0 { //New file, write header
|
||||||
csvw.Write([]string{
|
csvw.Write([]string{
|
||||||
@ -90,7 +101,7 @@ func (d *db) Save(records []Record) error {
|
|||||||
r.FeedEntry.Link,
|
r.FeedEntry.Link,
|
||||||
string(r.Status),
|
string(r.Status),
|
||||||
"-",
|
"-",
|
||||||
r.FeedEntry.Content,
|
strings.ReplaceAll(r.FeedEntry.Content, "\n", " "),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
@ -3,11 +3,11 @@ package app
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
|
"go.balki.me/tss/log"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -28,7 +28,7 @@ func NewScheduler(filePath string) (*Scheduler, error) {
|
|||||||
return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
|
return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
|
||||||
}
|
}
|
||||||
f.Close()
|
f.Close()
|
||||||
log.Printf("file: %v does not exist. Will be created\n", filePath)
|
log.Info("scheduler file does not exist, will be created", "path", filePath)
|
||||||
} else {
|
} else {
|
||||||
err = yaml.Unmarshal(data, &s.lastSuccessTime)
|
err = yaml.Unmarshal(data, &s.lastSuccessTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -26,13 +26,14 @@ type telegramSender struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (ts *telegramSender) SendLink(link string, channel string, rhash string) error {
|
func (ts *telegramSender) SendLink(link string, channel string, rhash string) error {
|
||||||
|
|
||||||
msg := struct {
|
msg := struct {
|
||||||
ChatID string `json:"chat_id"`
|
ChatID string `json:"chat_id"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
|
ParseMode string `json:"parse_mode"`
|
||||||
}{
|
}{
|
||||||
ChatID: channel,
|
ChatID: channel,
|
||||||
Text: fmt.Sprintf(`<a href="%s">➢</a> <a href="%s">Link</a>`, genIVLink(link, rhash), link),
|
Text: fmt.Sprintf(`<a href="%s">➢</a> <a href="%s">Link</a>`, genIVLink(link, rhash), link),
|
||||||
|
ParseMode: "HTML",
|
||||||
}
|
}
|
||||||
data, err := json.Marshal(msg)
|
data, err := json.Marshal(msg)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user