From f5e462845957950158f9536206adef6623bb6b40 Mon Sep 17 00:00:00 2001
From: balki <3070606-balki@users.noreply.gitlab.com>
Date: Sun, 1 May 2022 23:28:54 -0400
Subject: [PATCH] mvp before debug
---
app/app.go | 73 +++++++++++++++++++++-------------------
app/config.go | 14 ++++----
exp/ref/main.go | 9 +++++
main.go | 46 ++++---------------------
proxy/proxy.go | 12 +++----
telegram/telegram.go | 80 ++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 148 insertions(+), 86 deletions(-)
create mode 100644 telegram/telegram.go
diff --git a/app/app.go b/app/app.go
index 28cc250..71ade10 100644
--- a/app/app.go
+++ b/app/app.go
@@ -2,9 +2,12 @@ package app
import (
"io"
+ "net/http"
+ "time"
"go.balki.me/tss/log"
"go.balki.me/tss/proxy"
+ "go.balki.me/tss/telegram"
)
func Run(configPath string) {
@@ -15,6 +18,9 @@ func Run(configPath string) {
}
scheduler, err := NewScheduler(cfg.LastSuccessPath)
+ if err != nil {
+ log.Panic("failed to create scheduler", "path", cfg.LastSuccessPath, "error", err)
+ }
defer func() {
if err := scheduler.Save(); err != nil {
@@ -22,45 +28,20 @@ func Run(configPath string) {
}
}()
- // tgram := NewTelegramSender(cfg.TelegramProxy)
+ tgramProxy, err := proxy.GetTransport(cfg.TelegramProxy)
+ if err != nil {
+ log.Panic("failed to get proxy transport", "proxyURL", cfg.TelegramProxy, "error", err)
+ }
+
+ tgram := telegram.NewTelegramSender(tgramProxy, cfg.TelegramAuthToken)
for _, feed := range cfg.Feeds {
log.Info("processing feed", "feed", feed.Name)
- ProcessFeed(feed, scheduler, cfg.DbDir)
+ ProcessFeed(feed, scheduler, cfg.DbDir, tgram)
}
-
- /*
- for _, feed := range cfg.Feeds {
- log.Println("Processing feed", feed.Name)
- data, err := Download(feed.Url, feed.Proxy)
- if err != nil {
- log.Fatal(err)
- }
- }
-
- for _, feed := range c.Feeds {
- log.Println("Processing feed", feed.Name)
-
- links, err := parseFeed(data)
- if err != nil {
- log.Fatal(err)
- }
-
- for _, link := range links {
- if alreadySent(link) {
- continue
- }
- err := sendTelegram(link, feed.Channel, feed.Rhash)
- if err != nil {
- log.Fatal(err)
- }
- }
- }
- fmt.Println(configPath)
- */
}
-func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string) {
+func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string, tgram telegram.TelegramSender) {
sd, err := scheduler.ShouldDownload(feed.Name, feed.Cron)
if err != nil {
log.Error("shouldDownload failed", "feed", feed.Name, "err", err)
@@ -90,17 +71,39 @@ func ProcessFeed(feed FeedCfg, scheduler *Scheduler, dbDir string) {
return
}
- _, err = db.Filter(entries)
+ filteredEntries, err := db.Filter(entries)
if err != nil {
log.Error("failed to filter entries", "feed", feed.Name, "error", err)
}
+
+ var records []Record
+ for _, entry := range filteredEntries {
+ now := time.Now()
+ r := Record{
+ Time: now,
+ FeedEntry: entry,
+ }
+ err := tgram.SendLink(entry.Link, feed.Channel, feed.Rhash)
+ if err != nil {
+ log.Error("failed to send to telegram", "feed", feed.Name, "link", entry.Link, "channel", feed.Channel, "rhash", feed.Rhash, "error", err)
+ r.Status = Error
+ } else {
+ r.Status = Sent
+ }
+ records = append(records, r)
+ }
+ err = db.Save(records)
+ if err != nil {
+ log.Error("failed to save sent records", "feed", feed.Name, "num_records", len(records), "error", err)
+ }
}
func Download(url string, proxyUrl string) ([]byte, error) {
- client, err := proxy.GetClient(proxyUrl)
+ transport, err := proxy.GetTransport(proxyUrl)
if err != nil {
return nil, err
}
+ client := &http.Client{Transport: transport}
res, err := client.Get(url)
if err != nil {
return nil, err
diff --git a/app/config.go b/app/config.go
index c08159f..ce73040 100644
--- a/app/config.go
+++ b/app/config.go
@@ -17,12 +17,14 @@ type FeedCfg struct {
}
type Config struct {
- Proxy string `yaml:"proxy"`
- TelegramProxy string `yaml:"telegram_proxy"`
- DataDir string `yaml:"data_dir"`
- LastSuccessPath string `yaml:"last_loaded_path"`
- DbDir string `yaml:"db_dir"`
- Feeds []FeedCfg `yaml:"feeds"`
+ Proxy string `yaml:"proxy"`
+ TelegramProxy string `yaml:"telegram_proxy"`
+ //TODO: read from credential file
+ TelegramAuthToken string `yaml:"telegram_auth_token"`
+ DataDir string `yaml:"data_dir"`
+ LastSuccessPath string `yaml:"last_loaded_path"`
+ DbDir string `yaml:"db_dir"`
+ Feeds []FeedCfg `yaml:"feeds"`
}
func ParseConfig(configPath string) (*Config, error) {
diff --git a/exp/ref/main.go b/exp/ref/main.go
index 524be9c..6a1a648 100644
--- a/exp/ref/main.go
+++ b/exp/ref/main.go
@@ -23,4 +23,13 @@ func main() {
fmt.Println(dum.name)
}
+ foo()
+
+}
+
+func foo() {
+ var stuff []string
+ defer fmt.Println(stuff)
+ stuff = append(stuff, "foo")
+ stuff = append(stuff, "bar")
}
diff --git a/main.go b/main.go
index d8ec336..f474dc6 100644
--- a/main.go
+++ b/main.go
@@ -1,46 +1,14 @@
package main
import (
- "log"
- "net/url"
- "os"
+ "flag"
+
+ "go.balki.me/tss/app"
)
func main() {
- var err error
-
-}
-
-func download(url string) ([]byte, error) {
- log.Println("url", url)
- return os.ReadFile("ounapuu.xml")
-}
-
-func parseFeed(data []byte) ([]string, error) {
- return []string{"https://blog.link"}, nil
-}
-
-func alreadySent(link string) bool {
- return false
-}
-
-func sendTelegram(link string, channel string, rhash string) error {
- log.Println("link", link, "channel", channel, "rhash", rhash)
- rhash = "ae86262f2de32f"
- log.Println("ivurl", genIVLink(link, rhash))
-
- return nil
-}
-
-func genIVLink(link, rhash string) string {
- query := url.Values{}
- query.Set("url", link)
- query.Set("rhash", rhash)
- u := url.URL{
- Scheme: "https",
- Host: "t.me",
- Path: "iv",
- RawQuery: vs.Encode(),
- }
- return u.String()
+ var configPath string
+ flag.StringVar(&configPath, "config", "./tss.yml", "path to tss.yml config")
+ flag.Parse()
+ app.Run(configPath)
}
diff --git a/proxy/proxy.go b/proxy/proxy.go
index 8919484..9dc2d6b 100644
--- a/proxy/proxy.go
+++ b/proxy/proxy.go
@@ -8,9 +8,9 @@ import (
"golang.org/x/net/proxy"
)
-func GetClient(proxy string) (*http.Client, error) {
+func GetTransport(proxy string) (http.RoundTripper, error) {
if proxy == "" {
- return http.DefaultClient, nil
+ return http.DefaultTransport, nil
}
proxyUrl, err := url.Parse(proxy)
if err != nil {
@@ -23,7 +23,7 @@ func GetClient(proxy string) (*http.Client, error) {
return proxyHttp(proxyUrl)
}
-func unixSocks5Proxy(path string) (*http.Client, error) {
+func unixSocks5Proxy(path string) (http.RoundTripper, error) {
// TODO: Auth?
dialer, err := proxy.SOCKS5("unix", path, nil /*auth*/, nil)
if err != nil {
@@ -36,13 +36,13 @@ func unixSocks5Proxy(path string) (*http.Client, error) {
trans := defaultTransport()
trans.DialContext = ctxDialer.DialContext
trans.Proxy = nil
- return &http.Client{Transport: trans}, nil
+ return trans, nil
}
-func proxyHttp(proxyUrl *url.URL) (*http.Client, error) {
+func proxyHttp(proxyUrl *url.URL) (http.RoundTripper, error) {
trans := defaultTransport()
trans.Proxy = http.ProxyURL(proxyUrl)
- return &http.Client{Transport: trans}, nil
+ return trans, nil
}
func defaultTransport() *http.Transport {
diff --git a/telegram/telegram.go b/telegram/telegram.go
new file mode 100644
index 0000000..d49efa6
--- /dev/null
+++ b/telegram/telegram.go
@@ -0,0 +1,80 @@
+package telegram
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+
+ "go.balki.me/tss/log"
+)
+
+func main() {
+ fmt.Println("vim-go")
+}
+
+type TelegramSender interface {
+ SendLink(link, channel, rhash string) error
+}
+
+type telegramSender struct {
+ client *http.Client
+ authToken string
+}
+
+func (ts *telegramSender) SendLink(link string, channel string, rhash string) error {
+
+ msg := struct {
+ ChatID string `json:"chat_id"`
+ Text string `json:"text"`
+ }{
+ ChatID: channel,
+ Text: fmt.Sprintf(`➢ Link`, genIVLink(link, rhash), link),
+ }
+ data, err := json.Marshal(msg)
+
+ if err != nil {
+ return err
+ }
+
+ apiUrl := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", ts.authToken)
+ res, err := ts.client.Post(apiUrl, "application/json", bytes.NewReader(data))
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+ responseText, err := io.ReadAll(res.Body)
+ if err != nil {
+ return err
+ }
+ if res.StatusCode != http.StatusOK {
+ log.Error("telegram send failed", "status", res.Status, "request", data, "response", responseText)
+ return errors.New("telegram send failed")
+ }
+ log.Info("sent message on telegram", "link", link, "channel", channel, "response", responseText)
+ return nil
+}
+
+func NewTelegramSender(transport http.RoundTripper, authToken string) TelegramSender {
+ //TODO: Rate limit
+ return &telegramSender{
+ client: &http.Client{Transport: transport},
+ authToken: authToken,
+ }
+}
+
+func genIVLink(link, rhash string) string {
+ query := url.Values{}
+ query.Set("url", link)
+ query.Set("rhash", rhash)
+ u := url.URL{
+ Scheme: "https",
+ Host: "t.me",
+ Path: "iv",
+ RawQuery: query.Encode(),
+ }
+ return u.String()
+}