From 33e54a068b7ab2fa8ed51898770d2d245f36c338 Mon Sep 17 00:00:00 2001 From: Martin Dosch Date: Fri, 13 Jul 2018 22:53:22 +0200 Subject: [PATCH] Initial commit --- .gitignore | 17 +++++ LICENSE | 9 +++ README.md | 47 ++++++++++++ config.json.example | 10 +++ feed-to-muc.go | 152 +++++++++++++++++++++++++++++++++++++ getarticles.go | 178 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 413 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config.json.example create mode 100644 feed-to-muc.go create mode 100644 getarticles.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1138d6f --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +config.json +.vscode + +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bd1fe40 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright 2018 Martin Dosch + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ab722db --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +## about + +*feed-to-muc* is a XMPP which queries Atom or RSS newsfeeds and +posts the short summary to a XMPP MUC if there is a new article. + +## disclaimer + +I am no programmer and this bot is a result of a lot of *try and error*. +There are probably lots of quirks that are remains of some wrong paths +I went in between. Also a lot is probably solved in a *hacky way* due +to missing experience. +Anyway, it works (for me at least) and so I upload this here. +Recommendations about what can be done better improved and so on are +very welcome. + +## requirements + +* [go](https://golang.org/) + +## installation + +If you have *[GOPATH](https://github.com/golang/go/wiki/SettingGOPATH)* +set just run this commands: + +``` +$ go get salsa.debian.org/mdosch-guest/feed-to-muc +$ go install salsa.debian.org/mdosch-guest/feed-to-muc +``` + +You will find the binary in `$GOPATH/bin` or, if set, `$GOBIN`. + +## configuration + +The configuration is expected at `$HOME/.config/gowttr/config.json` with this format: + +``` +{ +"ServerAddress": "example.com:5222", +"BotJid": "feedbot@example.com", +"Password": "ChangeThis!", +"Muc": "muc-to-feed@conference.example.com", +"MucNick": "feedbot", +"MaxArticles": 5, +"Feeds": [ "https://www.debian.org/News/news", + "https://www.debian.org/security/dsa-long" ] +} +``` diff --git a/config.json.example b/config.json.example new file mode 100644 index 0000000..79c2fbc --- /dev/null +++ b/config.json.example @@ -0,0 +1,10 @@ +{ +"ServerAddress": "example.com:5222", +"BotJid": "feedbot@example.com", +"Password": "ChangeThis!", +"Muc": "muc-to-feed@conference.example.com", +"MucNick": "feedbot", +"MaxArticles": 5, +"Feeds": [ "https://www.debian.org/News/news", + "https://www.debian.org/security/dsa-long" ] +} diff --git a/feed-to-muc.go b/feed-to-muc.go new file mode 100644 index 0000000..f64c9c3 --- /dev/null +++ b/feed-to-muc.go @@ -0,0 +1,152 @@ +/* Copyright 2018 Martin Dosch +Licensed under the "MIT License" */ + +package main + +import ( + "encoding/json" + "log" + "os" + "os/user" + "strings" + "time" + + "github.com/mattn/go-xmpp" +) + +func main() { + + var err error + var configPath string + + type configuration struct { + ServerAddress string + BotJid string + Password string + Muc string + MucNick string + MaxArticles int + Feeds []string + } + + // Get systems user config path. + osConfigDir := os.Getenv("$XDG_CONFIG_HOME") + if osConfigDir != "" { + // Create configPath if not yet existing. + configPath = osConfigDir + "/.config/feed-to-muc/" + if _, err := os.Stat(configPath); os.IsNotExist(err) { + err = os.MkdirAll(configPath, 0700) + if err != nil { + log.Fatal("Error: ", err) + } + } + + } else { // Get the current user. + curUser, err := user.Current() + if err != nil { + log.Fatal("Error: ", err) + return + } + // Get home directory. + home := curUser.HomeDir + + if home == "" { + log.Fatal("Error: No home directory available.") + return + } + + // Create configPath if not yet existing. + configPath = home + "/.config/feed-to-muc/" + if _, err := os.Stat(configPath + "config.json"); os.IsNotExist(err) { + err = os.MkdirAll(configPath, 0700) + if err != nil { + log.Fatal("Error: ", err) + } + } + + } + + // Check that config file is existing. + if _, err := os.Stat(configPath + "config.json"); os.IsNotExist(err) { + log.Fatal("Error: ", err) + } + + // Read configuration file into variable config. + file, _ := os.Open(configPath + "config.json") + defer file.Close() + decoder := json.NewDecoder(file) + config := configuration{} + if err := decoder.Decode(&config); err != nil { + log.Fatal("Error: ", err) + } + + if _, err := os.Stat(configPath + "config.json"); os.IsNotExist(err) { + err = os.MkdirAll(configPath, 0700) + if err != nil { + log.Fatal("Error: ", err) + } + } + + var client *xmpp.Client + + options := xmpp.Options{ + Host: config.ServerAddress, + User: config.BotJid, + Password: config.Password, + NoTLS: true, + StartTLS: true, + Debug: false, + } + + client, err = options.NewClient() + if err != nil { + log.Fatal(err) + } + + // Starting goroutine to ping the server every 30 seconds. + go ping(client, config.ServerAddress, config.BotJid) + + // Join the MUC + mucStatus, err := client.JoinMUCNoHistory(config.Muc, config.MucNick) + if err != nil { + log.Fatal(err) + } + + // Exit if Status is > 300, see https://xmpp.org/registrar/mucstatus.html + if mucStatus > 300 { + os.Exit(mucStatus) + } + + for { + _, err := client.Recv() + if err != nil { + log.Fatal(err) + } + + for i := 0; i < len(config.Feeds); i++ { + output, err := getArticles(config.Feeds[i], config.MaxArticles, configPath) + if err != nil { + log.Fatal(err) + } + + if output != "" { + _, err = client.Send(xmpp.Chat{Remote: config.Muc, Type: "groupchat", Text: output}) + } + if err != nil { + log.Fatal(err) + } + } + time.Sleep(30 * time.Second) + } +} + +// Send a ping every 30 seconds to check if the server is still available. +func ping(client *xmpp.Client, server string, botJid string) { + for { + time.Sleep(30 * time.Second) + err := client.PingC2S(botJid, strings.Split(server, ":")[0]) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/getarticles.go b/getarticles.go new file mode 100644 index 0000000..39fbeed --- /dev/null +++ b/getarticles.go @@ -0,0 +1,178 @@ +/* Copyright 2018 Martin Dosch +Licensed under the "MIT License" */ + +package main + +import ( + "encoding/json" + "hash/fnv" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/mmcdole/gofeed" + "jaytaylor.com/html2text" +) + +//func getArticles(max int) ([]string, error) { +func getArticles(feedURL string, max int, cachePath string) (string, error) { + + type feedCache struct { + LastChange string + } + + var output string + var last time.Time + var lastUpdate feedCache + var file *os.File + var updateTime time.Time + + // Create a hash as identifier for the feed. + // The identifier will be used as filename for caching the update time. + // ToDo: cachePath should probably be moved to ~/.cache/feed-to-muc + h := fnv.New32a() + h.Write([]byte(feedURL)) + cachePath = cachePath + "cache/" + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + err = os.MkdirAll(cachePath, 0700) + if err != nil { + log.Fatal("Error: ", err) + } + } + + cacheFile := cachePath + strconv.Itoa(int(h.Sum32())) + + if _, err := os.Stat(cacheFile); os.IsNotExist(err) { + file, err = os.Create(cacheFile) + if err != nil { + log.Fatal("Error: ", err) + } + defer file.Close() + + last = time.Now() + + lastUpdate.LastChange = last.Format(time.RFC3339) + + lastUpdateJSON, _ := json.MarshalIndent(lastUpdate, "", " ") + _, err = file.Write(lastUpdateJSON) + if err != nil { + log.Fatal("Error: ", err) + } + + } else { + + file, err = os.OpenFile(cacheFile, os.O_RDWR, 0600) + if err != nil { + log.Fatal("Error: ", err) + } + defer file.Close() + + decoder := json.NewDecoder(file) + lastUpdate := feedCache{} + if err := decoder.Decode(&lastUpdate); err != nil { + log.Fatal("Error: ", err) + } + + last, err = time.Parse(time.RFC3339, string(lastUpdate.LastChange)) + if err != nil { + log.Fatal("Error: ", err) + } + } + + fp := gofeed.NewParser() + feed, err := fp.ParseURL(feedURL) + if err != nil { + return "", err + } + + // If no publish date is offered try update date. + // If both is not offered give up. + if feed.Items[0].PublishedParsed == nil { + if feed.Items[0].UpdatedParsed == nil { + return "", err + } + // If cached timestamp is newer than the one of + // the last article return. + if last.After(*feed.Items[0].UpdatedParsed) { + return "", err + } + } else { + // If cached timestamp is newer than the one of + // the last article return. + if last.After(*feed.Items[0].PublishedParsed) { + return "", err + } + } + + // Check last n (defined in config) articles for new ones. + for i := max - 1; i >= 0; i-- { + // Stop processing for article i if there are not so + // many articles in the feed. + if len(feed.Items) < i+1 { + continue + } + article := *feed.Items[i] + if err != nil { + return "", err + } + + if article.PublishedParsed == nil { + updateTime = *article.UpdatedParsed + } else { + updateTime = *article.PublishedParsed + } + + // If cached timestamp is not older than the article stop processing. + // Note: Checking for cached timestamp being newer, instead of not older + // lead to duplicate messages for the same article. Probably a corner + // case when the time is identical. + if last.Before(updateTime) == false { + continue + } + + if i == 0 { + last = updateTime + lastUpdate.LastChange = updateTime.Format(time.RFC3339) + + // Remove file with cached timestamp and create it + // again with updated timestamp. + // ToDo: Replace timestamp without deleting. + err = os.Remove(cacheFile) + if err != nil { + log.Fatal("Error: ", err) + } + + file, err = os.Create(cacheFile) + if err != nil { + log.Fatal("Error: ", err) + } + defer file.Close() + + lastUpdateJSON, _ := json.MarshalIndent(lastUpdate, "", " ") + _, err = file.Write(lastUpdateJSON) + if err != nil { + log.Fatal("Error: ", err) + } + } + + // Strip HTML as we want to get plain text. + description, err := html2text.FromString(article.Description) + if err != nil { + return "", err + } + + if strings.Contains(description, article.Link) == true { + output = output + feed.Title + ": *" + article.Title + "*\n\n" + description + } else { + output = output + feed.Title + ": *" + article.Title + "*\n\n" + description + "\n" + article.Link + } + + if i > 0 { + output = output + "\n\n---\n\n" + } + } + + return output, err +}