98 lines
2.3 KiB
Go
98 lines
2.3 KiB
Go
package fetch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/mmcdole/gofeed"
|
|
|
|
"github.com/ytshih/stonks/internal/config"
|
|
)
|
|
|
|
func init() {
|
|
Register("rss", newRSSFetcher)
|
|
}
|
|
|
|
// rssParams is the plugin-specific config for the rss fetcher.
|
|
type rssParams struct {
|
|
Feeds []string `yaml:"feeds"`
|
|
// MaxItemsPerFeed caps how many of the newest entries are kept per feed.
|
|
// Defaults to 10.
|
|
MaxItemsPerFeed int `yaml:"max_items_per_feed"`
|
|
// MaxAgeHours drops entries older than this many hours. Defaults to 48.
|
|
// Feeds with entries lacking a published date are kept regardless.
|
|
MaxAgeHours int `yaml:"max_age_hours"`
|
|
}
|
|
|
|
// rssFetcher pulls general market news headlines from a configured list of
|
|
// RSS feeds. Free, no API key.
|
|
type rssFetcher struct {
|
|
feeds []string
|
|
maxItemsPerFeed int
|
|
maxAge time.Duration
|
|
parser *gofeed.Parser
|
|
}
|
|
|
|
func newRSSFetcher(cfg config.PluginConfig) (Fetcher, error) {
|
|
params := rssParams{
|
|
MaxItemsPerFeed: 10,
|
|
MaxAgeHours: 48,
|
|
}
|
|
if err := cfg.DecodeParams(¶ms); err != nil {
|
|
return nil, err
|
|
}
|
|
if len(params.Feeds) == 0 {
|
|
return nil, fmt.Errorf("rss: at least one feed URL must be configured")
|
|
}
|
|
|
|
return &rssFetcher{
|
|
feeds: params.Feeds,
|
|
maxItemsPerFeed: params.MaxItemsPerFeed,
|
|
maxAge: time.Duration(params.MaxAgeHours) * time.Hour,
|
|
parser: gofeed.NewParser(),
|
|
}, nil
|
|
}
|
|
|
|
func (f *rssFetcher) Name() string { return "rss" }
|
|
|
|
func (f *rssFetcher) Fetch(ctx context.Context, _ config.Targets) (Result, error) {
|
|
var result Result
|
|
var errs []error
|
|
cutoff := time.Now().Add(-f.maxAge)
|
|
|
|
for _, feedURL := range f.feeds {
|
|
feed, err := f.parser.ParseURLWithContext(feedURL, ctx)
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("rss %s: %w", feedURL, err))
|
|
continue
|
|
}
|
|
|
|
count := 0
|
|
for _, item := range feed.Items {
|
|
if count >= f.maxItemsPerFeed {
|
|
break
|
|
}
|
|
if item.PublishedParsed != nil && item.PublishedParsed.Before(cutoff) {
|
|
continue
|
|
}
|
|
|
|
var published time.Time
|
|
if item.PublishedParsed != nil {
|
|
published = *item.PublishedParsed
|
|
}
|
|
|
|
result.News = append(result.News, NewsItem{
|
|
Title: item.Title,
|
|
Source: feed.Title,
|
|
URL: item.Link,
|
|
Published: published,
|
|
Summary: item.Description,
|
|
})
|
|
count++
|
|
}
|
|
}
|
|
|
|
return result, joinErrs(errs)
|
|
}
|