more progress

app/app.go

@@ -1,12 +1,28 @@
 package app
 
-func Run(configPath string) {
-	/*
-		cfg, err := ParseConfig(configPath)
-		if err != nil {
-			log.Fatal(err)
-		}
+import "go.balki.me/tss/log"
 
+func Run(configPath string) {
+	cfg, err := ParseConfig(configPath)
+
+	if err != nil {
+		log.Panic("failed to parse config", "path", configPath, "err", err)
+	}
+
+	scheduler, err := NewScheduler(cfg.LastSuccessPath)
+
+	defer func() {
+		if err := scheduler.Save(); err != nil {
+			log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err)
+		}
+	}()
+
+	for _, feed := range cfg.Feeds {
+		log.Info("processing feed", "feed", feed.Name)
+		ProcessFeed(feed, scheduler)
+	}
+
+	/*
 		for _, feed := range cfg.Feeds {
 			log.Println("Processing feed", feed.Name)
 			data, err := Download(feed.Url, feed.Proxy)
@@ -36,3 +52,20 @@ func Run(configPath string) {
 		fmt.Println(configPath)
 	*/
 }
+
+func ProcessFeed(feed FeedCfg, scheduler *Scheduler) {
+	sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
+	if err != nil {
+		log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
+		return
+	}
+	if !sd {
+		log.Info("skipping feed due to schedule", "feed", feed.Name)
+		return
+	}
+	_, err = Download(feed.Url, feed.Proxy)
+	if err != nil {
+		log.Error("download failed", "feed", feed.Name, "url", feed.Url, "proxy", feed.Proxy)
+		return
+	}
+}

app/config.go

@@ -2,22 +2,28 @@ package app
 
 import (
 	"os"
+	"path"
 
 	"gopkg.in/yaml.v3"
 )
 
+type FeedCfg struct {
+	Name    string `yaml:"name"`
+	Channel string `yaml:"channel"`
+	Rhash   string `yaml:"rhash"`
+	Url     string `yaml:"url"`
+	Cron    string `yaml:"cron"`
+	Proxy   string `yaml:"proxy"`
+}
+
 type Config struct {
-	Feeds []struct {
-		Name    string `yaml:"name"`
-		Channel string `yaml:"channel"`
-		Rhash   string `yaml:"rhash"`
-		Url     string `yaml:"url"`
-		Cron    string `yaml:"cron"`
-		Proxy   string `yaml:"proxy"`
-	} `yaml:"feeds"`
+	DataDir         string    `yaml:"data_dir"`
+	LastSuccessPath string    `yaml:"last_loaded_path"`
+	Feeds           []FeedCfg `yaml:"feeds"`
 }
 
 func ParseConfig(configPath string) (*Config, error) {
+
 	cfg, err := os.ReadFile(configPath)
 	if err != nil {
 		return nil, err
@@ -30,5 +36,13 @@ func ParseConfig(configPath string) (*Config, error) {
 		return nil, err
 	}
 
+	if c.DataDir == "" {
+		c.DataDir = "."
+	}
+
+	if c.LastSuccessPath == "" {
+		c.LastSuccessPath = path.Join(c.DataDir, "last_success.yml")
+	}
+
 	return &c, nil
 }

app/config_test.go

@@ -0,0 +1,41 @@
+package app
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
+)
+
+var contents = []byte(`
+data_dir: blah
+feeds:
+  - url: "https://balki.me"
+    name: "Blah"
+  - url: "https://blue.balki.me"
+    name: "sldkfj Blah"
+`)
+
+func TestConfigParse(t *testing.T) {
+	c := Config{}
+	err := yaml.Unmarshal(contents, &c)
+	assert.NoError(t, err)
+	assert.Equal(t, "blah", c.DataDir)
+	/*
+	   yesterday := time.Now().Add(-24 * time.Hour)
+	   m := map[string]time.Time{
+	           "Bala":  time.Now(),
+	           "Linus": yesterday,
+	   }
+	   out, err := yaml.Marshal(&m)
+	   if err != nil {
+	           t.Fatalf("got err: %v", err)
+	   }
+	   // assert.Equal(t, nil, fmt.Sprintf("%s", out))
+	   var m2 map[string]time.Time
+	   err = yaml.Unmarshal(out, &m2)
+	   assert.NoError(t, err)
+
+	   assert.Equal(t, "foobar", fmt.Sprint(m2["Bala"]))
+	*/
+}

app/schedule.go

@@ -1,6 +1,9 @@
 package app
 
 import (
+	"errors"
+	"fmt"
+	"log"
 	"os"
 	"time"
 
@@ -17,11 +20,20 @@ func NewScheduler(filePath string) (*Scheduler, error) {
 	s := Scheduler{filePath: filePath}
 	data, err := os.ReadFile(filePath)
 	if err != nil {
-		return nil, err
-	}
-	err = yaml.Unmarshal(data, &s.lastSuccessTime)
-	if err != nil {
-		return nil, err
+		if !errors.Is(err, os.ErrNotExist) {
+			return nil, err
+		}
+		f, err := os.Create(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
+		}
+		f.Close()
+		log.Printf("file: %v does not exist. Will be created\n", filePath)
+	} else {
+		err = yaml.Unmarshal(data, &s.lastSuccessTime)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return &s, nil
 }