Skip to content

Instantly share code, notes, and snippets.

@mojocn

mojocn/main.go

Last active February 10, 2018 04:13
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save mojocn/9b18db2c99b01e49ce6afbbb2322e07a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save mojocn/9b18db2c99b01e49ce6afbbb2322e07a to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. mojocn revised this gist Feb 10, 2018. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions main.go
    View file
    Original file line number Diff line number Diff line change
    @@ -14,9 +14,9 @@ import (
    var (
    redisClient *redis.Client
    //公司大群
    dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=651f4847c3425a7a8f4346383643bfda89cd4b991422b256d2acdee58a51e115"
    dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=tokentokentoken"
    //软件群
    //dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=caa4dd76839b2413045a13099945af87bb7e84924678657afb3564b653b8918d"
    //dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=tokentokentoken"

    baiduNewsUrls = []string{
    "http://news.baidu.com/ns?cl=2&rn=20&tn=news&word=%E7%89%A9%E8%81%94%E7%BD%91", //物联网
  2. mojocn created this gist Feb 10, 2018.
    136 changes: 136 additions & 0 deletions main.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,136 @@
    package main

    import (
    "bytes"
    "fmt"
    "github.com/PuerkitoBio/goquery"
    "github.com/astaxie/beego/toolbox"
    "github.com/go-redis/redis"
    "log"
    "net/http"
    "strings"
    )

    var (
    redisClient *redis.Client
    //公司大群
    dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=651f4847c3425a7a8f4346383643bfda89cd4b991422b256d2acdee58a51e115"
    //软件群
    //dingdingURL = "https://oapi.dingtalk.com/robot/send?access_token=caa4dd76839b2413045a13099945af87bb7e84924678657afb3564b653b8918d"

    baiduNewsUrls = []string{
    "http://news.baidu.com/ns?cl=2&rn=20&tn=news&word=%E7%89%A9%E8%81%94%E7%BD%91", //物联网
    "http://news.baidu.com/ns?word=LoRa&tn=news&from=news&cl=2&rn=20&ct=1", //lora
    "http://news.baidu.com/ns?ct=1&rn=20&ie=utf-8&bs=NB-IoT&rsv_bp=1&sr=0&cl=2&f=8&prevct=no&tn=news&word=NB-IoT", //nb-iot
    "http://news.baidu.com/ns?word=%E6%99%BA%E6%85%A7%E6%B6%88%E9%98%B2&tn=news&from=news&cl=2&rn=20&ct=1", //智慧消防
    "http://news.baidu.com/ns?word=%E6%97%A0%E7%BA%BF%E6%B6%88%E9%98%B2&tn=news&from=news&cl=2&rn=20&ct=1", //无线消防
    "http://news.baidu.com/ns?word=%E6%99%BA%E6%85%A7%E5%81%9C%E8%BD%A6&tn=news&from=news&cl=2&rn=20&ct=1", //智慧停车
    }
    )

    const (
    newsFeed = "news_feed"
    newsPost = "news_post"
    newsList = "iot_news"
    )

    func init() {
    redisClient = redis.NewClient(&redis.Options{
    Addr: "127.0.0.1:6379",
    Password: "8796534shdjq384wejgkguiern", // no password set
    DB: 2, // use default DB
    })
    }

    func main() {
    defer redisClient.Close()
    //NewsBot()
    //创建定时任务
    dingdingNewBot := toolbox.NewTask("dingding-news-bot", "0 30 8 * * *", NewsBot)
    //dingdingNewBot := toolbox.NewTask("dingding-news-bot", "0 40 */1 * * *", NewsBot)
    //err := dingdingNewBot.Run()
    //检测定时任务
    // if err != nil {
    // log.Fatal(err)
    // }
    //添加定时任务
    toolbox.AddTask("dingding-news-bot", dingdingNewBot)
    //启动定时任务
    toolbox.StartTask()
    defer toolbox.StopTask()
    select {}
    }

    func spiderNewsIntoRedisPipeLine(url string, pipe redis.Pipeliner) error {
    doc, err := goquery.NewDocument(url)
    if err != nil {
    return nil
    }
    // Find the review items
    doc.Find("div.result").Each(func(i int, s *goquery.Selection) {
    // For each item found, get the band and title
    URL, _ := s.Find("h3 > a").Attr("href")
    Source := s.Find("p.c-author").Text()
    Title := s.Find("h3 > a").Text()
    markdown := fmt.Sprintf("- [%s](%s) _%s_", Title, URL, Source)
    pipe.HSet(newsList, URL, markdown)
    pipe.SAdd(newsFeed, URL)
    })
    return nil
    }

    func NewsBot() error {
    //爬去新闻
    pipe := redisClient.Pipeline()
    for _, baiduSearchUrl := range baiduNewsUrls {
    spiderNewsIntoRedisPipeLine(baiduSearchUrl, pipe)
    }
    pipe.Exec()
    //redis sdiff 找出未发送的文章
    unSendNewsUrls := redisClient.SDiff(newsFeed, newsPost).Val()
    //获取未发送文章的标题
    unSentNews := redisClient.HMGet(newsList, unSendNewsUrls...).Val()
    //每条消息显示20条
    batchCount := 20

    tempArray := make([]string, batchCount)
    newsCount := len(unSentNews)
    for idx, value := range unSentNews {
    tempArray[idx%batchCount] = value.(string)
    //分组发送新闻
    if (idx > 0 && idx%batchCount == 0) || newsCount-1 == idx {
    content := strings.Join(tempArray, "\n")
    log.Println(content)
    sendDingDingMsg(content)
    tempArray = make([]string, batchCount)
    }
    }
    //标记已发送的文章url
    unPostUrls := make([]interface{}, len(unSendNewsUrls))
    for key, url := range unSendNewsUrls {
    unPostUrls[key] = url
    }
    _, err := redisClient.SAdd(newsPost, unPostUrls...).Result()

    log.Println(err)
    return err

    }

    func sendDingDingMsg(content string) error {
    if content == "" {
    return nil
    }
    formt := `
    {
    "msgtype": "markdown",
    "markdown": {
    "title":"拓宝舆情监控",
    "text": "%s"
    }
    }`
    body := fmt.Sprintf(formt, content)
    jsonValue := []byte(body)
    _, err := http.Post(dingdingURL, "application/json", bytes.NewBuffer(jsonValue))
    return err
    }