From 715f25ec5cc137341d43ba21f505fee3c16a4c44 Mon Sep 17 00:00:00 2001 From: balki <3070606-balki@users.noreply.gitlab.com> Date: Thu, 28 Apr 2022 19:24:21 -0400 Subject: [PATCH] more progress --- TODO.md | 13 ++++++------ app/app.go | 45 +++++++++++++++++++++++++++++++++------ app/config.go | 30 +++++++++++++++++++------- app/config_test.go | 41 +++++++++++++++++++++++++++++++++++ app/schedule.go | 22 ++++++++++++++----- exp/file/file.go | 29 +++++++++++++++++++++++++ exp/log/main.go | 17 +++++++++++++++ go.mod | 7 ++++-- go.sum | 53 ++++++++++++++++++++++++++++++++++++++++++++-- log/exp.go | 1 + log/log.go | 35 ++++++++++++++++++++++++++++++ 11 files changed, 264 insertions(+), 29 deletions(-) create mode 100644 app/config_test.go create mode 100644 exp/file/file.go create mode 100644 exp/log/main.go create mode 100644 log/exp.go create mode 100644 log/log.go diff --git a/TODO.md b/TODO.md index de0bcae..82515c0 100644 --- a/TODO.md +++ b/TODO.md @@ -2,17 +2,18 @@ * ✓ Write TODO * ✓ Cron -* Config validation - * Cron spec - * Name - * Permissions? - -* logging https://github.com/go-logr/logr +* ✓ Config +* ✓ logging zap * Proxy download * Posting to telegram * Storage csv ## 1.0 +* concurrent processing +* Config validation + * Cron spec + * Name - No spaces + * Permissions? * Filtering * Doc embed * Doc diff --git a/app/app.go b/app/app.go index a418109..2b3a01e 100644 --- a/app/app.go +++ b/app/app.go @@ -1,12 +1,28 @@ package app -func Run(configPath string) { - /* - cfg, err := ParseConfig(configPath) - if err != nil { - log.Fatal(err) - } +import "go.balki.me/tss/log" +func Run(configPath string) { + cfg, err := ParseConfig(configPath) + + if err != nil { + log.Panic("failed to parse config", "path", configPath, "err", err) + } + + scheduler, err := NewScheduler(cfg.LastSuccessPath) + + defer func() { + if err := scheduler.Save(); err != nil { + log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err) + } + }() + + for _, feed := range cfg.Feeds { + log.Info("processing feed", "feed", feed.Name) + ProcessFeed(feed, scheduler) + } + + /* for _, feed := range cfg.Feeds { log.Println("Processing feed", feed.Name) data, err := Download(feed.Url, feed.Proxy) @@ -36,3 +52,20 @@ func Run(configPath string) { fmt.Println(configPath) */ } + +func ProcessFeed(feed FeedCfg, scheduler *Scheduler) { + sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron) + if err != nil { + log.Error("shouldDownload failed", "feed", feed.Name, "err", err) + return + } + if !sd { + log.Info("skipping feed due to schedule", "feed", feed.Name) + return + } + _, err = Download(feed.Url, feed.Proxy) + if err != nil { + log.Error("download failed", "feed", feed.Name, "url", feed.Url, "proxy", feed.Proxy) + return + } +} diff --git a/app/config.go b/app/config.go index dab6553..ba02325 100644 --- a/app/config.go +++ b/app/config.go @@ -2,22 +2,28 @@ package app import ( "os" + "path" "gopkg.in/yaml.v3" ) +type FeedCfg struct { + Name string `yaml:"name"` + Channel string `yaml:"channel"` + Rhash string `yaml:"rhash"` + Url string `yaml:"url"` + Cron string `yaml:"cron"` + Proxy string `yaml:"proxy"` +} + type Config struct { - Feeds []struct { - Name string `yaml:"name"` - Channel string `yaml:"channel"` - Rhash string `yaml:"rhash"` - Url string `yaml:"url"` - Cron string `yaml:"cron"` - Proxy string `yaml:"proxy"` - } `yaml:"feeds"` + DataDir string `yaml:"data_dir"` + LastSuccessPath string `yaml:"last_loaded_path"` + Feeds []FeedCfg `yaml:"feeds"` } func ParseConfig(configPath string) (*Config, error) { + cfg, err := os.ReadFile(configPath) if err != nil { return nil, err @@ -30,5 +36,13 @@ func ParseConfig(configPath string) (*Config, error) { return nil, err } + if c.DataDir == "" { + c.DataDir = "." + } + + if c.LastSuccessPath == "" { + c.LastSuccessPath = path.Join(c.DataDir, "last_success.yml") + } + return &c, nil } diff --git a/app/config_test.go b/app/config_test.go new file mode 100644 index 0000000..f9b8ae5 --- /dev/null +++ b/app/config_test.go @@ -0,0 +1,41 @@ +package app + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +var contents = []byte(` +data_dir: blah +feeds: + - url: "https://balki.me" + name: "Blah" + - url: "https://blue.balki.me" + name: "sldkfj Blah" +`) + +func TestConfigParse(t *testing.T) { + c := Config{} + err := yaml.Unmarshal(contents, &c) + assert.NoError(t, err) + assert.Equal(t, "blah", c.DataDir) + /* + yesterday := time.Now().Add(-24 * time.Hour) + m := map[string]time.Time{ + "Bala": time.Now(), + "Linus": yesterday, + } + out, err := yaml.Marshal(&m) + if err != nil { + t.Fatalf("got err: %v", err) + } + // assert.Equal(t, nil, fmt.Sprintf("%s", out)) + var m2 map[string]time.Time + err = yaml.Unmarshal(out, &m2) + assert.NoError(t, err) + + assert.Equal(t, "foobar", fmt.Sprint(m2["Bala"])) + */ +} diff --git a/app/schedule.go b/app/schedule.go index 9813999..fcce36a 100644 --- a/app/schedule.go +++ b/app/schedule.go @@ -1,6 +1,9 @@ package app import ( + "errors" + "fmt" + "log" "os" "time" @@ -17,11 +20,20 @@ func NewScheduler(filePath string) (*Scheduler, error) { s := Scheduler{filePath: filePath} data, err := os.ReadFile(filePath) if err != nil { - return nil, err - } - err = yaml.Unmarshal(data, &s.lastSuccessTime) - if err != nil { - return nil, err + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + f, err := os.Create(filePath) + if err != nil { + return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err) + } + f.Close() + log.Printf("file: %v does not exist. Will be created\n", filePath) + } else { + err = yaml.Unmarshal(data, &s.lastSuccessTime) + if err != nil { + return nil, err + } } return &s, nil } diff --git a/exp/file/file.go b/exp/file/file.go new file mode 100644 index 0000000..3ca5613 --- /dev/null +++ b/exp/file/file.go @@ -0,0 +1,29 @@ +package main + +import ( + "fmt" + "log" + "os" +) + +func main() { + fmt.Println("vim-go") + f, err := os.Create("./lsdkj/lfksdjf") + if err != nil { + log.Panicln(err) + } + err = f.Close() + if err != nil { + log.Panicln(err) + } + log.Println("created") + /* + data, err := os.ReadFile("./foo/notexist") + if err == nil { + fmt.Println(data) + } else if errors.Is(err, os.ErrNotExist) { + fmt.Printf("not exist: %v\n", err) + } + fmt.Printf("err: %v\n", err) + */ +} diff --git a/exp/log/main.go b/exp/log/main.go new file mode 100644 index 0000000..50489b5 --- /dev/null +++ b/exp/log/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "go.balki.me/tss/log" + "go.uber.org/zap" +) + +func main() { + fmt.Println("vim-go") + log.Info("does this work", "trycount", 1) + // fmt.Sprint("ldskfjdsice") + s := zap.S() + s.Warnw("How does this look", "trycount", 2) + s.Sync() +} diff --git a/go.mod b/go.mod index 1ef6605..0412b52 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,17 @@ module go.balki.me/tss -go 1.17 +go 1.18 require ( github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.7.1 + go.uber.org/zap v1.21.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) require ( - github.com/davecgh/go-spew v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 4b7a289..644c352 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,63 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/log/exp.go b/log/exp.go new file mode 100644 index 0000000..7330d54 --- /dev/null +++ b/log/exp.go @@ -0,0 +1 @@ +package log diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..2e48b6c --- /dev/null +++ b/log/log.go @@ -0,0 +1,35 @@ +package log + +import ( + "go.uber.org/zap" +) + +var Logger *zap.SugaredLogger + +var Debug, Info, Warn, Error, Panic func(msg string, keysAndValues ...interface{}) + +func init() { + + cfg := zap.NewProductionConfig() + devCfg := zap.NewDevelopmentConfig() + + // Readable time stamp + cfg.EncoderConfig.EncodeTime = devCfg.EncoderConfig.EncodeTime + + // Uncomment to enable debug logging + // cfg.Level = devCfg.Level + + logger, err := cfg.Build() + if err != nil { + println("unable to intialize zap log") + panic(err) + } + + Logger = logger.Sugar() + + Debug = Logger.Debugw + Info = Logger.Infow + Warn = Logger.Warnw + Error = Logger.Errorw + Panic = Logger.Panicw +}