more progress
This commit is contained in:
parent
9491c83f50
commit
715f25ec5c
13
TODO.md
13
TODO.md
@ -2,17 +2,18 @@
|
||||
|
||||
* ✓ Write TODO
|
||||
* ✓ Cron
|
||||
* Config validation
|
||||
* Cron spec
|
||||
* Name
|
||||
* Permissions?
|
||||
|
||||
* logging https://github.com/go-logr/logr
|
||||
* ✓ Config
|
||||
* ✓ logging zap
|
||||
* Proxy download
|
||||
* Posting to telegram
|
||||
* Storage csv
|
||||
|
||||
## 1.0
|
||||
* concurrent processing
|
||||
* Config validation
|
||||
* Cron spec
|
||||
* Name - No spaces
|
||||
* Permissions?
|
||||
* Filtering
|
||||
* Doc embed
|
||||
* Doc
|
||||
|
||||
45
app/app.go
45
app/app.go
@ -1,12 +1,28 @@
|
||||
package app
|
||||
|
||||
func Run(configPath string) {
|
||||
/*
|
||||
cfg, err := ParseConfig(configPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
import "go.balki.me/tss/log"
|
||||
|
||||
func Run(configPath string) {
|
||||
cfg, err := ParseConfig(configPath)
|
||||
|
||||
if err != nil {
|
||||
log.Panic("failed to parse config", "path", configPath, "err", err)
|
||||
}
|
||||
|
||||
scheduler, err := NewScheduler(cfg.LastSuccessPath)
|
||||
|
||||
defer func() {
|
||||
if err := scheduler.Save(); err != nil {
|
||||
log.Panic("failed to save last success info", "path", cfg.LastSuccessPath, "err", err)
|
||||
}
|
||||
}()
|
||||
|
||||
for _, feed := range cfg.Feeds {
|
||||
log.Info("processing feed", "feed", feed.Name)
|
||||
ProcessFeed(feed, scheduler)
|
||||
}
|
||||
|
||||
/*
|
||||
for _, feed := range cfg.Feeds {
|
||||
log.Println("Processing feed", feed.Name)
|
||||
data, err := Download(feed.Url, feed.Proxy)
|
||||
@ -36,3 +52,20 @@ func Run(configPath string) {
|
||||
fmt.Println(configPath)
|
||||
*/
|
||||
}
|
||||
|
||||
func ProcessFeed(feed FeedCfg, scheduler *Scheduler) {
|
||||
sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
|
||||
if err != nil {
|
||||
log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
|
||||
return
|
||||
}
|
||||
if !sd {
|
||||
log.Info("skipping feed due to schedule", "feed", feed.Name)
|
||||
return
|
||||
}
|
||||
_, err = Download(feed.Url, feed.Proxy)
|
||||
if err != nil {
|
||||
log.Error("download failed", "feed", feed.Name, "url", feed.Url, "proxy", feed.Proxy)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,22 +2,28 @@ package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
type FeedCfg struct {
|
||||
Name string `yaml:"name"`
|
||||
Channel string `yaml:"channel"`
|
||||
Rhash string `yaml:"rhash"`
|
||||
Url string `yaml:"url"`
|
||||
Cron string `yaml:"cron"`
|
||||
Proxy string `yaml:"proxy"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Feeds []struct {
|
||||
Name string `yaml:"name"`
|
||||
Channel string `yaml:"channel"`
|
||||
Rhash string `yaml:"rhash"`
|
||||
Url string `yaml:"url"`
|
||||
Cron string `yaml:"cron"`
|
||||
Proxy string `yaml:"proxy"`
|
||||
} `yaml:"feeds"`
|
||||
DataDir string `yaml:"data_dir"`
|
||||
LastSuccessPath string `yaml:"last_loaded_path"`
|
||||
Feeds []FeedCfg `yaml:"feeds"`
|
||||
}
|
||||
|
||||
func ParseConfig(configPath string) (*Config, error) {
|
||||
|
||||
cfg, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -30,5 +36,13 @@ func ParseConfig(configPath string) (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if c.DataDir == "" {
|
||||
c.DataDir = "."
|
||||
}
|
||||
|
||||
if c.LastSuccessPath == "" {
|
||||
c.LastSuccessPath = path.Join(c.DataDir, "last_success.yml")
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
41
app/config_test.go
Normal file
41
app/config_test.go
Normal file
@ -0,0 +1,41 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var contents = []byte(`
|
||||
data_dir: blah
|
||||
feeds:
|
||||
- url: "https://balki.me"
|
||||
name: "Blah"
|
||||
- url: "https://blue.balki.me"
|
||||
name: "sldkfj Blah"
|
||||
`)
|
||||
|
||||
func TestConfigParse(t *testing.T) {
|
||||
c := Config{}
|
||||
err := yaml.Unmarshal(contents, &c)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "blah", c.DataDir)
|
||||
/*
|
||||
yesterday := time.Now().Add(-24 * time.Hour)
|
||||
m := map[string]time.Time{
|
||||
"Bala": time.Now(),
|
||||
"Linus": yesterday,
|
||||
}
|
||||
out, err := yaml.Marshal(&m)
|
||||
if err != nil {
|
||||
t.Fatalf("got err: %v", err)
|
||||
}
|
||||
// assert.Equal(t, nil, fmt.Sprintf("%s", out))
|
||||
var m2 map[string]time.Time
|
||||
err = yaml.Unmarshal(out, &m2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "foobar", fmt.Sprint(m2["Bala"]))
|
||||
*/
|
||||
}
|
||||
@ -1,6 +1,9 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
@ -17,11 +20,20 @@ func NewScheduler(filePath string) (*Scheduler, error) {
|
||||
s := Scheduler{filePath: filePath}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = yaml.Unmarshal(data, &s.lastSuccessTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("path:%v does not exist and unable to create: err: %w", filePath, err)
|
||||
}
|
||||
f.Close()
|
||||
log.Printf("file: %v does not exist. Will be created\n", filePath)
|
||||
} else {
|
||||
err = yaml.Unmarshal(data, &s.lastSuccessTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
29
exp/file/file.go
Normal file
29
exp/file/file.go
Normal file
@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("vim-go")
|
||||
f, err := os.Create("./lsdkj/lfksdjf")
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
log.Println("created")
|
||||
/*
|
||||
data, err := os.ReadFile("./foo/notexist")
|
||||
if err == nil {
|
||||
fmt.Println(data)
|
||||
} else if errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Printf("not exist: %v\n", err)
|
||||
}
|
||||
fmt.Printf("err: %v\n", err)
|
||||
*/
|
||||
}
|
||||
17
exp/log/main.go
Normal file
17
exp/log/main.go
Normal file
@ -0,0 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"go.balki.me/tss/log"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("vim-go")
|
||||
log.Info("does this work", "trycount", 1)
|
||||
// fmt.Sprint("ldskfjdsice")
|
||||
s := zap.S()
|
||||
s.Warnw("How does this look", "trycount", 2)
|
||||
s.Sync()
|
||||
}
|
||||
7
go.mod
7
go.mod
@ -1,14 +1,17 @@
|
||||
module go.balki.me/tss
|
||||
|
||||
go 1.17
|
||||
go 1.18
|
||||
|
||||
require (
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/stretchr/testify v1.7.1
|
||||
go.uber.org/zap v1.21.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
)
|
||||
|
||||
53
go.sum
53
go.sum
@ -1,14 +1,63 @@
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
|
||||
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
1
log/exp.go
Normal file
1
log/exp.go
Normal file
@ -0,0 +1 @@
|
||||
package log
|
||||
35
log/log.go
Normal file
35
log/log.go
Normal file
@ -0,0 +1,35 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var Logger *zap.SugaredLogger
|
||||
|
||||
var Debug, Info, Warn, Error, Panic func(msg string, keysAndValues ...interface{})
|
||||
|
||||
func init() {
|
||||
|
||||
cfg := zap.NewProductionConfig()
|
||||
devCfg := zap.NewDevelopmentConfig()
|
||||
|
||||
// Readable time stamp
|
||||
cfg.EncoderConfig.EncodeTime = devCfg.EncoderConfig.EncodeTime
|
||||
|
||||
// Uncomment to enable debug logging
|
||||
// cfg.Level = devCfg.Level
|
||||
|
||||
logger, err := cfg.Build()
|
||||
if err != nil {
|
||||
println("unable to intialize zap log")
|
||||
panic(err)
|
||||
}
|
||||
|
||||
Logger = logger.Sugar()
|
||||
|
||||
Debug = Logger.Debugw
|
||||
Info = Logger.Infow
|
||||
Warn = Logger.Warnw
|
||||
Error = Logger.Errorw
|
||||
Panic = Logger.Panicw
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user