add doc for schema generation

app/config.go

@@ -16,27 +16,65 @@ const NoLimit FeedLimit = -1
 type FeedURL []string
 
 type FeedCfg struct {
-	Name           string          `yaml:"name"`
+	// Unique identifier for the feed. Used as filename, so keep it simple
-	Channel        string          `yaml:"channel"`
+	Name string `yaml:"name" jsonschema:"required"`
-	Rhash          string          `yaml:"rhash"`
+
-	Url            FeedURL         `yaml:"url"`
+	// Telegram channel name in the form "@foobar" or "-1004242442" (private)
-	Cron           string          `yaml:"cron"`
+	Channel string `yaml:"channel" jsonschema:"required"`
-	Proxy          string          `yaml:"proxy"`
+
-	Type           parser.FeedType `yaml:"type"`
+	// Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates
-	Title          string          `yaml:"title"`
+	Rhash string `yaml:"rhash" jsonschema:"default="`
-	FTL            *int            `yaml:"first_time_limit"`
+
-	FirstTimeLimit FeedLimit       `yaml:"-"`
+	// Feed Url, string or array of strings
+	// If array, all entries are merged, e.g ["https://example.com/blog/tag/foo/feed", "https://example.com/blog/tag/bar/feed" ]
+	Url FeedURL `yaml:"url" jsonschema:"required,oneof_type=string;array"`
+
+	// Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format
+	Cron string `yaml:"cron" jsonschema:"required"`
+
+	// Proxy for this feed if different from global
+	Proxy string `yaml:"proxy" jsonschema:"default=<global proxy>"`
+
+	// Rss or Atom feed
+	Type parser.FeedType `yaml:"type" jsonschema:"required,enum=rss,enum=atom"`
+
+	// Text of the link sent to telegram,
+	Title string `yaml:"title" jsonschema:"default=Link"`
+
+	// Number of entries to process when the feed is seen for the first time
+	FTL *int `yaml:"first_time_limit" jsonschema:"default=<global value>"`
+
+	FirstTimeLimit FeedLimit `yaml:"-"`
 }
 
 type Config struct {
-	Proxy             string    `yaml:"proxy"`
+	// default proxy url that http.Client can understand
-	TelegramProxy     string    `yaml:"telegram_proxy"`
+	// or socks5://unix/path/to/sock for socks proxy over unix socket
-	TelegramAuthToken string    `yaml:"telegram_auth_token"` //TODO: read from credential file
+	// empty for no proxy
-	DataDir           string    `yaml:"data_dir"`
+	Proxy string `yaml:"proxy" jsonschema:"default="`
-	LastSuccessPath   string    `yaml:"last_loaded_path"`
+
-	DbDir             string    `yaml:"db_dir"`
+	// proxy for telegram if different from default
-	FTL               *int      `yaml:"first_time_limit"`
+	// NONE to not use global default
-	Feeds             []FeedCfg `yaml:"feeds"`
+	TelegramProxy string `yaml:"telegram_proxy" jsonschema:"default=<global proxy>"`
+
+	// Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX
+	TelegramAuthToken string `yaml:"telegram_auth_token" jsonschema:"required"` //TODO: read from credential file
+
+	// Base directory to write feed data and schedule file
+	DataDir string `yaml:"data_dir" jsonschema:"default=./"`
+
+	// Path to the file where last success time for each feed is saved
+	LastSuccessPath string `yaml:"last_loaded_path" jsonschema:"default=<datadir>/last_success.yml"`
+
+	// Path to the directory where feed content is saved
+	DbDir string `yaml:"db_dir" jsonschema:"default=<datadir>/feed_data"`
+
+	// Number of entries to process when the feed is seen for the first time
+	// -1 means no limit
+	FTL *int `yaml:"first_time_limit" jsonschema:"default=-1"`
+
+	// Feed config
+	Feeds []FeedCfg `yaml:"feeds" jsonschema:"required"`
 }
 
 func ParseConfig(configPath string) (*Config, error) {

cmd/gen_schema/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/invopop/jsonschema"
+	"go.balki.me/tss/app"
+)
+
+func fix(s string) string {
+	// s := "go.balki.me/tss/home/balki/projects/tss/app.Config.Proxy"
+	return fmt.Sprintf("%s%s", s[:16], s[40:])
+}
+
+type config struct {
+	app.Config
+
+	// Placeholder to put any yaml anchors to use elsewhere, E.g.
+	// refs:
+	//   myvpsproxy: &myvpsproxy "socks5://vps.example.com:1080"
+	//   twiceaday: &twiceaday "00 1,13 * * *"
+	Refs map[string]any `yaml:"refs" jsonschema:"refs"`
+}
+
+func main() {
+	// "go.balki.me/tss/app.FeedCfg.Channel": "Telegram channel name in the form \"@foobar\" or \"-1004242442\"",
+	//s go.balki.me/tss/app.FeedCfg.Channel
+	// foo()
+	// return
+	fmt.Println("vim-go")
+	cmmap := map[string]string{}
+	err := jsonschema.ExtractGoComments("go.balki.me/tss", "/home/balki/projects/tss", cmmap)
+	cmmapF := map[string]string{}
+	for k, v := range cmmap {
+		cmmapF[fix(k)] = v
+	}
+	cmmap = cmmapF
+	fmt.Println("err", err)
+	d, err := json.Marshal(cmmap)
+	fmt.Printf("%s\n", string(d))
+	os.WriteFile("/tmp/cmmap.json", d, 0644)
+	var c config
+	r := jsonschema.Reflector{
+		RequiredFromJSONSchemaTags: true,
+		DoNotReference:             true,
+		ExpandedStruct:             true,
+		CommentMap:                 cmmap,
+	}
+	s := r.Reflect(c)
+	j, _ := json.Marshal(s)
+	os.WriteFile("/home/balki/projects/tss/docs/schema.json", j, 0644)
+	fmt.Println(string(j))
+}

docs/schema.json

@@ -0,0 +1 @@
+{"$schema":"http://json-schema.org/draft/2020-12/schema","properties":{"proxy":{"type":"string","description":"default proxy url that http.Client can understand\nor socks5://unix/path/to/sock for socks proxy over unix socket\nempty for no proxy","default":""},"telegram_proxy":{"type":"string","description":"proxy for telegram if different from default\nNONE to not use global default","default":"\u003cglobal proxy\u003e"},"telegram_auth_token":{"type":"string","description":"Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX"},"data_dir":{"type":"string","description":"Base directory to write feed data and schedule file","default":"./"},"last_loaded_path":{"type":"string","description":"Path to the file where last success time for each feed is saved","default":"\u003cdatadir\u003e/last_success.yml"},"db_dir":{"type":"string","description":"Path to the directory where feed content is saved","default":"\u003cdatadir\u003e/feed_data"},"first_time_limit":{"type":"integer","description":"Number of entries to process when the feed is seen for the first time\n-1 means no limit","default":-1},"feeds":{"items":{"properties":{"name":{"type":"string","description":"Unique identifier for the feed. Used as filename, so keep it simple"},"channel":{"type":"string","description":"Telegram channel name in the form \"@foobar\" or \"-1004242442\" (private)"},"rhash":{"type":"string","description":"Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates","default":""},"url":{"oneOf":[{"type":"string"},{"type":"array"}],"items":{"type":"string"},"description":"Feed Url, string or array of strings\nIf array, all entries are merged, e.g [\"https://example.com/blog/tag/foo/feed\", \"https://example.com/blog/tag/bar/feed\" ]"},"cron":{"type":"string","description":"Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format"},"proxy":{"type":"string","description":"Proxy for this feed if different from global","default":"\u003cglobal proxy\u003e"},"type":{"type":"string","enum":["rss","atom"],"description":"Rss or Atom feed"},"title":{"type":"string","description":"Text of the link sent to telegram,","default":"Link"},"first_time_limit":{"type":"integer","description":"Number of entries to process when the feed is seen for the first time","default":0}},"additionalProperties":false,"type":"object","required":["name","channel","url","cron","type"]},"type":"array","description":"Feed config"},"refs":{"type":"object"}},"additionalProperties":false,"type":"object","required":["telegram_auth_token","feeds"]}

docs/schema.yaml

@@ -0,0 +1,96 @@
+$schema: http://json-schema.org/draft/2020-12/schema
+properties:
+  proxy:
+    type: string
+    description: |-
+      default proxy url that http.Client can understand
+      or socks5://unix/path/to/sock for socks proxy over unix socket
+      empty for no proxy
+    default: ""
+  telegram_proxy:
+    type: string
+    description: |-
+      proxy for telegram if different from default
+      NONE to not use global default
+    default: <global proxy>
+  telegram_auth_token:
+    type: string
+    description: Auth token for telegram in the form of 424242424:AB02132skdcjlisjkjflewjlfkjslkfHHXX
+  data_dir:
+    type: string
+    description: Base directory to write feed data and schedule file
+    default: ./
+  last_loaded_path:
+    type: string
+    description: Path to the file where last success time for each feed is saved
+    default: <datadir>/last_success.yml
+  db_dir:
+    type: string
+    description: Path to the directory where feed content is saved
+    default: <datadir>/feed_data
+  first_time_limit:
+    type: integer
+    description: |-
+      Number of entries to process when the feed is seen for the first time
+      -1 means no limit
+    default: -1
+  feeds:
+    items:
+      properties:
+        name:
+          type: string
+          description: Unique identifier for the feed. Used as filename, so keep it simple
+        channel:
+          type: string
+          description: Telegram channel name in the form "@foobar" or "-1004242442" (private)
+        rhash:
+          type: string
+          description: Telegram instant view rhash id, see https://instantview.telegram.org/#publishing-templates
+          default: ""
+        url:
+          oneOf:
+            - type: string
+            - type: array
+          items:
+            type: string
+          description: |-
+            Feed Url, string or array of strings
+            If array, all entries are merged, e.g ["https://example.com/blog/tag/foo/feed", "https://example.com/blog/tag/bar/feed" ]
+        cron:
+          type: string
+          description: "Cron expression, see https://pkg.go.dev/github.com/robfig/cron#hdr-CRON_Expression_Format"
+        proxy:
+          type: string
+          description: Proxy for this feed if different from global
+          default: <global proxy>
+        type:
+          type: string
+          enum:
+            - rss
+            - atom
+          description: Rss or Atom feed
+        title:
+          type: string
+          description: Text of the link sent to telegram,
+          default: Link
+        first_time_limit:
+          type: integer
+          description: Number of entries to process when the feed is seen for the first time
+          default: 0
+      additionalProperties: false
+      type: object
+      required:
+        - name
+        - channel
+        - url
+        - cron
+        - type
+    type: array
+    description: Feed config
+  refs:
+    type: object
+additionalProperties: false
+type: object
+required:
+  - telegram_auth_token
+  - feeds

go.mod

@@ -5,6 +5,7 @@ go 1.18
 require (
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
+	github.com/invopop/jsonschema v0.4.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/stretchr/testify v1.7.3
 	go.uber.org/zap v1.21.0
@@ -14,6 +15,7 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect

go.sum

@@ -8,6 +8,10 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
 github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
+github.com/invopop/jsonschema v0.4.0 h1:Yuy/unfgCnfV5Wl7H0HgFufp/rlurqPOOuacqyByrws=
+github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -23,6 +27,7 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=