implement atom

app/app.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"go.balki.me/tss/log"
+	"go.balki.me/tss/parser"
 	"go.balki.me/tss/proxy"
 	"go.balki.me/tss/telegram"
 )
@@ -65,7 +66,7 @@ func ProcessFeed(feed FeedCfg, scheduler Scheduler, dbDir string, tgram telegram
 		return
 	}
 
-	entries, err := ParseFeed(data)
+	entries, err := parser.ParseFeed(feed.Type, data)
 	if err != nil {
 		log.Error("feed parsing failed", "feed", feed.Name, "data", data, "error", err)
 		return
@@ -74,13 +75,13 @@ func ProcessFeed(feed FeedCfg, scheduler Scheduler, dbDir string, tgram telegram
 	scheduler.Good(feed.Name)
 
 	var records []Record
-	var newEntries []FeedEntry
+	var newEntries []parser.FeedEntry
 	if db.IsNewFeed() {
 		ftl := int(feed.FirstTimeLimit)
 		if feed.FirstTimeLimit == NoLimit || len(entries) <= ftl {
 			newEntries = entries
 		} else {
-			var filteredEntries []FeedEntry
+			var filteredEntries []parser.FeedEntry
 			newEntries, filteredEntries = entries[:ftl], entries[ftl:]
 			for _, entry := range filteredEntries {
 				records = append(records, Record{

app/config.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"path"
 
+	"go.balki.me/tss/parser"
 	"gopkg.in/yaml.v3"
 )
 
@@ -12,14 +13,15 @@ type FeedLimit int
 const NoLimit FeedLimit = -1
 
 type FeedCfg struct {
-	Name           string    `yaml:"name"`
-	Channel        string    `yaml:"channel"`
-	Rhash          string    `yaml:"rhash"`
-	Url            string    `yaml:"url"`
-	Cron           string    `yaml:"cron"`
-	Proxy          string    `yaml:"proxy"`
-	FTL            *int      `yaml:"first_time_limit"`
-	FirstTimeLimit FeedLimit `yaml:"-"`
+	Name           string          `yaml:"name"`
+	Channel        string          `yaml:"channel"`
+	Rhash          string          `yaml:"rhash"`
+	Url            string          `yaml:"url"`
+	Cron           string          `yaml:"cron"`
+	Proxy          string          `yaml:"proxy"`
+	Type           parser.FeedType `yaml:"type"`
+	FTL            *int            `yaml:"first_time_limit"`
+	FirstTimeLimit FeedLimit       `yaml:"-"`
 }
 
 type Config struct {

app/db.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"go.balki.me/tss/log"
+	"go.balki.me/tss/parser"
 )
 
 type Status string
@@ -27,12 +28,12 @@ type Record struct {
 	Time      time.Time
 	Status    Status
 	Filter    string
-	FeedEntry FeedEntry
+	FeedEntry parser.FeedEntry
 }
 
 type DB interface {
 	IsNewFeed() bool
-	Filter(entries []FeedEntry) ([]FeedEntry, error)
+	Filter(entries []parser.FeedEntry) ([]parser.FeedEntry, error)
 	Save([]Record) error
 }
 
@@ -72,8 +73,8 @@ func NewDB(storageDir, feedName string) (DB, error) {
 	return &db, nil
 }
 
-func (d *db) Filter(entries []FeedEntry) ([]FeedEntry, error) {
-	var filteredEntries []FeedEntry
+func (d *db) Filter(entries []parser.FeedEntry) ([]parser.FeedEntry, error) {
+	var filteredEntries []parser.FeedEntry
 	for _, entry := range entries {
 		if _, ok := d.seenLinks[entry.Link]; !ok {
 			filteredEntries = append(filteredEntries, entry)

app/parser.go

@@ -1,28 +0,0 @@
-package app
-
-import (
-	"encoding/xml"
-)
-
-type FeedEntry struct {
-	Title       string `xml:"title"`
-	Link        string `xml:"link"`
-	Author      string `xml:"author"`
-	Guid        string `xml:"guid"`
-	Description string `xml:"description"`
-	Content     string `xml:",innerxml"`
-}
-
-func ParseFeed(data []byte) ([]FeedEntry, error) {
-
-	v := struct {
-		Items []FeedEntry `xml:"channel>item"`
-	}{}
-
-	err := xml.Unmarshal(data, &v)
-	if err != nil {
-		return nil, err
-	}
-
-	return v.Items, nil
-}

exp/xml/xml.go

@@ -29,6 +29,10 @@ type Item struct {
 
 func main() {
 	fmt.Println("hello go")
+	foo()
+	if 1 == 1 {
+		return
+	}
 	data, err := os.ReadFile("/home/balki/projects/tss/w.tmp/ounapuu.xml")
 	if err != nil {
 		log.Fatal(err)
@@ -93,11 +97,18 @@ func main() {
 func foo() {
 	x := `
 <foo>
-	<bar><blue>slkdfjdslk</blue></bar>
+	<bar src="hello">
+	<link href="http://foobar.com">Hello World</link>
+	<blue>slkdfjdslk</blue>
+	</bar>
 </foo>
 `
 	s := struct {
 		Bar struct {
+			Link struct {
+				Href string `xml:"href,attr"`
+			} `xml:"link"`
+			Attr    string `xml:"src,attr"`
 			Content string `xml:",innerxml"`
 		} `xml:"bar"`
 	}{}
@@ -106,5 +117,5 @@ func foo() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Println(s.Bar.Content)
+	fmt.Println(s.Bar.Link.Href)
 }

parser/parser.go

@@ -0,0 +1,77 @@
+package parser
+
+import (
+	"encoding/xml"
+	"fmt"
+)
+
+type FeedType string
+
+const (
+	Rss  FeedType = "rss"
+	Atom FeedType = "atom"
+)
+
+type FeedEntry struct {
+	Link    string
+	Content string
+}
+
+func ParseFeed(feedType FeedType, data []byte) ([]FeedEntry, error) {
+	switch feedType {
+	case Rss:
+		return parseRss(data)
+	case Atom:
+		return parseAtom(data)
+	}
+	return nil, fmt.Errorf("Unknown feed type: %s", feedType)
+}
+
+func parseAtom(data []byte) ([]FeedEntry, error) {
+	v := struct {
+		Items []struct {
+			Link struct {
+				Href string `xml:"href,attr"`
+			} `xml:"link"`
+			Content string `xml:",innerxml"`
+		} `xml:"entry"`
+	}{}
+
+	err := xml.Unmarshal(data, &v)
+	if err != nil {
+		return nil, err
+	}
+
+	var feedEntries []FeedEntry
+	for _, atomEntry := range v.Items {
+		feedEntries = append(feedEntries, FeedEntry{
+			Link:    atomEntry.Link.Href,
+			Content: atomEntry.Content,
+		})
+	}
+	return feedEntries, nil
+}
+func parseRss(data []byte) ([]FeedEntry, error) {
+
+	v := struct {
+		Items []struct {
+			Link    string `xml:"link"`
+			Content string `xml:",innerxml"`
+		} `xml:"channel>item"`
+	}{}
+
+	err := xml.Unmarshal(data, &v)
+	if err != nil {
+		return nil, err
+	}
+
+	var feedEntries []FeedEntry
+	for _, rssEntry := range v.Items {
+		feedEntries = append(feedEntries, FeedEntry{
+			Link:    rssEntry.Link,
+			Content: rssEntry.Content,
+		})
+	}
+
+	return feedEntries, nil
+}