balki/tss
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more progress

Browse Source
This commit is contained in:
Balakrishnan Balasubramanian 2022-04-28 19:24:21 -04:00
parent 9491c83f50
commit 715f25ec5c
11 changed files with 264 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
TODO.md
View File

@ -2,17 +2,18 @@
* ✓ Write TODO * ✓ Write TODO
* ✓ Cron * ✓ Cron
* Config validation * ✓ Config
* Cron spec * ✓ logging zap
* Name
* Permissions?
* logging https://github.com/go-logr/logr
* Proxy download * Proxy download
* Posting to telegram * Posting to telegram
* Storage csv * Storage csv
## 1.0 ## 1.0
* concurrent processing
* Config validation
* Cron spec
* Name - No spaces
* Permissions?
* Filtering * Filtering
* Doc embed * Doc embed
* Doc * Doc

45
app/app.go
View File

@ -1,12 +1,28 @@
package app package app
func Run(configPath string) { import "go.balki.me/tss/log"
/*
cfg, err := ParseConfig(configPath)
if err != nil {
log.Fatal(err)
}
func Run(configPath string) {
cfg, err := ParseConfig(configPath)
if err != nil {
log.Panic("failed to parse config", "path", configPath, "err", err)
}
scheduler, err := NewScheduler(cfg.LastSuccessPath)
defer func() {
if err := scheduler.Save(); err != nil {
log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err)
}
}()
for _, feed := range cfg.Feeds {
log.Info("processing feed", "feed", feed.Name)
ProcessFeed(feed, scheduler)
}
/*
for _, feed := range cfg.Feeds { for _, feed := range cfg.Feeds {
log.Println("Processing feed", feed.Name) log.Println("Processing feed", feed.Name)
data, err := Download(feed.Url, feed.Proxy) data, err := Download(feed.Url, feed.Proxy)
@ -36,3 +52,20 @@ func Run(configPath string) {
fmt.Println(configPath) fmt.Println(configPath)
*/ */
} }
func ProcessFeed(feed FeedCfg, scheduler *Scheduler) {
sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
if err != nil {
log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
return
}
if !sd {
log.Info("skipping feed due to schedule", "feed", feed.Name)
return
}
_, err = Download(feed.Url, feed.Proxy)
if err != nil {
log.Error("download failed", "feed", feed.Name, "url", feed.Url, "proxy", feed.Proxy)
return
}
}

30
app/config.go
View File

@ -2,22 +2,28 @@ package app
import ( import (
"os" "os"
"path"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
type FeedCfg struct {
Name string `yaml:"name"`
Channel string `yaml:"channel"`
Rhash string `yaml:"rhash"`
Url string `yaml:"url"`
Cron string `yaml:"cron"`
Proxy string `yaml:"proxy"`
}
type Config struct { type Config struct {
Feeds []struct { DataDir string `yaml:"data_dir"`
Name string `yaml:"name"` LastSuccessPath string `yaml:"last_loaded_path"`
Channel string `yaml:"channel"` Feeds []FeedCfg `yaml:"feeds"`
Rhash string `yaml:"rhash"`
Url string `yaml:"url"`
Cron string `yaml:"cron"`
Proxy string `yaml:"proxy"`
} `yaml:"feeds"`
} }
func ParseConfig(configPath string) (*Config, error) { func ParseConfig(configPath string) (*Config, error) {
cfg, err := os.ReadFile(configPath) cfg, err := os.ReadFile(configPath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -30,5 +36,13 @@ func ParseConfig(configPath string) (*Config, error) {
return nil, err return nil, err
} }
if c.DataDir == "" {
c.DataDir = "."
}
if c.LastSuccessPath == "" {
c.LastSuccessPath = path.Join(c.DataDir, "last_success.yml")
}
return &c, nil return &c, nil
} }

41
app/config_test.go Normal file
View File

@ -0,0 +1,41 @@
package app
import (
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
var contents = []byte(`
data_dir: blah
feeds:
- url: "https://balki.me"
name: "Blah"
- url: "https://blue.balki.me"
name: "sldkfj Blah"
`)
func TestConfigParse(t *testing.T) {
c := Config{}
err := yaml.Unmarshal(contents, &c)
assert.NoError(t, err)
assert.Equal(t, "blah", c.DataDir)
/*
yesterday := time.Now().Add(-24 * time.Hour)
m := map[string]time.Time{
"Bala": time.Now(),
"Linus": yesterday,
}
out, err := yaml.Marshal(&m)
if err != nil {
t.Fatalf("got err: %v", err)
}
// assert.Equal(t, nil, fmt.Sprintf("%s", out))
var m2 map[string]time.Time
err = yaml.Unmarshal(out, &m2)
assert.NoError(t, err)
assert.Equal(t, "foobar", fmt.Sprint(m2["Bala"]))
*/
}

22
app/schedule.go
View File

@ -1,6 +1,9 @@
package app package app
import ( import (
"errors"
"fmt"
"log"
"os" "os"
"time" "time"
@ -17,11 +20,20 @@ func NewScheduler(filePath string) (*Scheduler, error) {
s := Scheduler{filePath: filePath} s := Scheduler{filePath: filePath}
data, err := os.ReadFile(filePath) data, err := os.ReadFile(filePath)
if err != nil { if err != nil {
return nil, err if !errors.Is(err, os.ErrNotExist) {
} return nil, err
err = yaml.Unmarshal(data, &s.lastSuccessTime) }
if err != nil { f, err := os.Create(filePath)
return nil, err if err != nil {
return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
}
f.Close()
log.Printf("file: %v does not exist. Will be created\n", filePath)
} else {
err = yaml.Unmarshal(data, &s.lastSuccessTime)
if err != nil {
return nil, err
}
} }
return &s, nil return &s, nil
} }

29
exp/file/file.go Normal file
View File

@ -0,0 +1,29 @@
package main
import (
"fmt"
"log"
"os"
)
func main() {
fmt.Println("vim-go")
f, err := os.Create("./lsdkj/lfksdjf")
if err != nil {
log.Panicln(err)
}
err = f.Close()
if err != nil {
log.Panicln(err)
}
log.Println("created")
/*
data, err := os.ReadFile("./foo/notexist")
if err == nil {
fmt.Println(data)
} else if errors.Is(err, os.ErrNotExist) {
fmt.Printf("not exist: %v\n", err)
}
fmt.Printf("err: %v\n", err)
*/
}

17
exp/log/main.go Normal file
View File

@ -0,0 +1,17 @@
package main
import (
"fmt"
"go.balki.me/tss/log"
"go.uber.org/zap"
)
func main() {
fmt.Println("vim-go")
log.Info("does this work", "trycount", 1)
// fmt.Sprint("ldskfjdsice")
s := zap.S()
s.Warnw("How does this look", "trycount", 2)
s.Sync()
}

7
go.mod
View File

@ -1,14 +1,17 @@
module go.balki.me/tss module go.balki.me/tss
go 1.17 go 1.18
require ( require (
github.com/robfig/cron/v3 v3.0.1 github.com/robfig/cron/v3 v3.0.1
github.com/stretchr/testify v1.7.1 github.com/stretchr/testify v1.7.1
go.uber.org/zap v1.21.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
) )
require ( require (
github.com/davecgh/go-spew v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
) )

53
go.sum
View File

@ -1,14 +1,63 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

1
log/exp.go Normal file
View File

@ -0,0 +1 @@
package log

35
log/log.go Normal file
View File

@ -0,0 +1,35 @@
package log
import (
"go.uber.org/zap"
)
var Logger *zap.SugaredLogger
var Debug, Info, Warn, Error, Panic func(msg string, keysAndValues ...interface{})
func init() {
cfg := zap.NewProductionConfig()
devCfg := zap.NewDevelopmentConfig()
// Readable time stamp
cfg.EncoderConfig.EncodeTime = devCfg.EncoderConfig.EncodeTime
// Uncomment to enable debug logging
// cfg.Level = devCfg.Level
logger, err := cfg.Build()
if err != nil {
println("unable to intialize zap log")
panic(err)
}
Logger = logger.Sugar()
Debug = Logger.Debugw
Info = Logger.Infow
Warn = Logger.Warnw
Error = Logger.Errorw
Panic = Logger.Panicw
}