From a61be99d0369457e9e37df8c515bd228cfc42819 Mon Sep 17 00:00:00 2001 From: balki <3070606-balki@users.noreply.gitlab.com> Date: Mon, 20 Jun 2022 19:31:28 -0400 Subject: [PATCH] add doc for schema generation --- app/config.go | 74 ++++++++++++++++++++++++-------- cmd/gen_schema/main.go | 55 ++++++++++++++++++++++++ docs/schema.json | 1 + docs/schema.yaml | 96 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 + go.sum | 5 +++ 6 files changed, 215 insertions(+), 18 deletions(-) create mode 100644 cmd/gen_schema/main.go create mode 100644 docs/schema.json create mode 100644 docs/schema.yaml diff --git a/app/config.go b/app/config.go index abdf5d8..9dbe208 100644 --- a/app/config.go +++ b/app/config.go @@ -16,27 +16,65 @@ const NoLimit FeedLimit = -1 type FeedURL []string type FeedCfg struct { - Name string `yaml:"name"` - Channel string `yaml:"channel"` - Rhash string `yaml:"rhash"` - Url FeedURL `yaml:"url"` - Cron string `yaml:"cron"` - Proxy string `yaml:"proxy"` - Type parser.FeedType `yaml:"type"` - Title string `yaml:"title"` - FTL *int `yaml:"first_time_limit"` - FirstTimeLimit FeedLimit `yaml:"-"` + // Unique identifier for the feed. Used as filename, so keep it simple + Name string `yaml:"name" jsonschema:"required"` + + // Telegram channel name in the form "@foobar" or "-1004242442" (private) + Channel string `yaml:"channel" jsonschema:"required"` + + // Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates + Rhash string `yaml:"rhash" jsonschema:"default="` + + // Feed Url, string or array of strings + // If array, all entries are merged, e.g ["https://example.com/blog/tag/foo/feed", "https://example.com/blog/tag/bar/feed" ] + Url FeedURL `yaml:"url" jsonschema:"required,oneof_type=string;array"` + + // Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format + Cron string `yaml:"cron" jsonschema:"required"` + + // Proxy for this feed if different from global + Proxy string `yaml:"proxy" jsonschema:"default="` + + // Rss or Atom feed + Type parser.FeedType `yaml:"type" jsonschema:"required,enum=rss,enum=atom"` + + // Text of the link sent to telegram, + Title string `yaml:"title" jsonschema:"default=Link"` + + // Number of entries to process when the feed is seen for the first time + FTL *int `yaml:"first_time_limit" jsonschema:"default="` + + FirstTimeLimit FeedLimit `yaml:"-"` } type Config struct { - Proxy string `yaml:"proxy"` - TelegramProxy string `yaml:"telegram_proxy"` - TelegramAuthToken string `yaml:"telegram_auth_token"` //TODO: read from credential file - DataDir string `yaml:"data_dir"` - LastSuccessPath string `yaml:"last_loaded_path"` - DbDir string `yaml:"db_dir"` - FTL *int `yaml:"first_time_limit"` - Feeds []FeedCfg `yaml:"feeds"` + // default proxy url that http.Client can understand + // or socks5://unix/path/to/sock for socks proxy over unix socket + // empty for no proxy + Proxy string `yaml:"proxy" jsonschema:"default="` + + // proxy for telegram if different from default + // NONE to not use global default + TelegramProxy string `yaml:"telegram_proxy" jsonschema:"default="` + + // Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX + TelegramAuthToken string `yaml:"telegram_auth_token" jsonschema:"required"` //TODO: read from credential file + + // Base directory to write feed data and schedule file + DataDir string `yaml:"data_dir" jsonschema:"default=./"` + + // Path to the file where last success time for each feed is saved + LastSuccessPath string `yaml:"last_loaded_path" jsonschema:"default=/last_success.yml"` + + // Path to the directory where feed content is saved + DbDir string `yaml:"db_dir" jsonschema:"default=/feed_data"` + + // Number of entries to process when the feed is seen for the first time + // -1 means no limit + FTL *int `yaml:"first_time_limit" jsonschema:"default=-1"` + + // Feed config + Feeds []FeedCfg `yaml:"feeds" jsonschema:"required"` } func ParseConfig(configPath string) (*Config, error) { diff --git a/cmd/gen_schema/main.go b/cmd/gen_schema/main.go new file mode 100644 index 0000000..72dcf15 --- /dev/null +++ b/cmd/gen_schema/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/invopop/jsonschema" + "go.balki.me/tss/app" +) + +func fix(s string) string { + // s := "go.balki.me/tss/home/balki/projects/tss/app.Config.Proxy" + return fmt.Sprintf("%s%s", s[:16], s[40:]) +} + +type config struct { + app.Config + + // Placeholder to put any yaml anchors to use elsewhere, E.g. + // refs: + // myvpsproxy: &myvpsproxy "socks5://vps.example.com:1080" + // twiceaday: &twiceaday "00 1,13 * * *" + Refs map[string]any `yaml:"refs" jsonschema:"refs"` +} + +func main() { + // "go.balki.me/tss/app.FeedCfg.Channel": "Telegram channel name in the form \"@foobar\" or \"-1004242442\"", + //s go.balki.me/tss/app.FeedCfg.Channel + // foo() + // return + fmt.Println("vim-go") + cmmap := map[string]string{} + err := jsonschema.ExtractGoComments("go.balki.me/tss", "/home/balki/projects/tss", cmmap) + cmmapF := map[string]string{} + for k, v := range cmmap { + cmmapF[fix(k)] = v + } + cmmap = cmmapF + fmt.Println("err", err) + d, err := json.Marshal(cmmap) + fmt.Printf("%s\n", string(d)) + os.WriteFile("/tmp/cmmap.json", d, 0644) + var c config + r := jsonschema.Reflector{ + RequiredFromJSONSchemaTags: true, + DoNotReference: true, + ExpandedStruct: true, + CommentMap: cmmap, + } + s := r.Reflect(c) + j, _ := json.Marshal(s) + os.WriteFile("/home/balki/projects/tss/docs/schema.json", j, 0644) + fmt.Println(string(j)) +} diff --git a/docs/schema.json b/docs/schema.json new file mode 100644 index 0000000..33fb52e --- /dev/null +++ b/docs/schema.json @@ -0,0 +1 @@ +{"$schema":"http://json-schema.org/draft/2020-12/schema","properties":{"proxy":{"type":"string","description":"default proxy url that http.Client can understand\nor socks5://unix/path/to/sock for socks proxy over unix socket\nempty for no proxy","default":""},"telegram_proxy":{"type":"string","description":"proxy for telegram if different from default\nNONE to not use global default","default":"\u003cglobal proxy\u003e"},"telegram_auth_token":{"type":"string","description":"Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX"},"data_dir":{"type":"string","description":"Base directory to write feed data and schedule file","default":"./"},"last_loaded_path":{"type":"string","description":"Path to the file where last success time for each feed is saved","default":"\u003cdatadir\u003e/last_success.yml"},"db_dir":{"type":"string","description":"Path to the directory where feed content is saved","default":"\u003cdatadir\u003e/feed_data"},"first_time_limit":{"type":"integer","description":"Number of entries to process when the feed is seen for the first time\n-1 means no limit","default":-1},"feeds":{"items":{"properties":{"name":{"type":"string","description":"Unique identifier for the feed. Used as filename, so keep it simple"},"channel":{"type":"string","description":"Telegram channel name in the form \"@foobar\" or \"-1004242442\" (private)"},"rhash":{"type":"string","description":"Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates","default":""},"url":{"oneOf":[{"type":"string"},{"type":"array"}],"items":{"type":"string"},"description":"Feed Url, string or array of strings\nIf array, all entries are merged, e.g [\"https://example.com/blog/tag/foo/feed\", \"https://example.com/blog/tag/bar/feed\" ]"},"cron":{"type":"string","description":"Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format"},"proxy":{"type":"string","description":"Proxy for this feed if different from global","default":"\u003cglobal proxy\u003e"},"type":{"type":"string","enum":["rss","atom"],"description":"Rss or Atom feed"},"title":{"type":"string","description":"Text of the link sent to telegram,","default":"Link"},"first_time_limit":{"type":"integer","description":"Number of entries to process when the feed is seen for the first time","default":0}},"additionalProperties":false,"type":"object","required":["name","channel","url","cron","type"]},"type":"array","description":"Feed config"},"refs":{"type":"object"}},"additionalProperties":false,"type":"object","required":["telegram_auth_token","feeds"]} \ No newline at end of file diff --git a/docs/schema.yaml b/docs/schema.yaml new file mode 100644 index 0000000..2c6ddba --- /dev/null +++ b/docs/schema.yaml @@ -0,0 +1,96 @@ +$schema: http://json-schema.org/draft/2020-12/schema +properties: + proxy: + type: string + description: |- + default proxy url that http.Client can understand + or socks5://unix/path/to/sock for socks proxy over unix socket + empty for no proxy + default: "" + telegram_proxy: + type: string + description: |- + proxy for telegram if different from default + NONE to not use global default + default: + telegram_auth_token: + type: string + description: Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX + data_dir: + type: string + description: Base directory to write feed data and schedule file + default: ./ + last_loaded_path: + type: string + description: Path to the file where last success time for each feed is saved + default: /last_success.yml + db_dir: + type: string + description: Path to the directory where feed content is saved + default: /feed_data + first_time_limit: + type: integer + description: |- + Number of entries to process when the feed is seen for the first time + -1 means no limit + default: -1 + feeds: + items: + properties: + name: + type: string + description: Unique identifier for the feed. Used as filename, so keep it simple + channel: + type: string + description: Telegram channel name in the form "@foobar" or "-1004242442" (private) + rhash: + type: string + description: Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates + default: "" + url: + oneOf: + - type: string + - type: array + items: + type: string + description: |- + Feed Url, string or array of strings + If array, all entries are merged, e.g ["https://example.com/blog/tag/foo/feed", "https://example.com/blog/tag/bar/feed" ] + cron: + type: string + description: "Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format" + proxy: + type: string + description: Proxy for this feed if different from global + default: + type: + type: string + enum: + - rss + - atom + description: Rss or Atom feed + title: + type: string + description: Text of the link sent to telegram, + default: Link + first_time_limit: + type: integer + description: Number of entries to process when the feed is seen for the first time + default: 0 + additionalProperties: false + type: object + required: + - name + - channel + - url + - cron + - type + type: array + description: Feed config + refs: + type: object +additionalProperties: false +type: object +required: + - telegram_auth_token + - feeds diff --git a/go.mod b/go.mod index 8f7d40c..535d3ed 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 + github.com/invopop/jsonschema v0.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.7.3 go.uber.org/zap v1.21.0 @@ -14,6 +15,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 8a8d4e3..1fa545e 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,10 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/invopop/jsonschema v0.4.0 h1:Yuy/unfgCnfV5Wl7H0HgFufp/rlurqPOOuacqyByrws= +github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,6 +27,7 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=