newsagent.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: itabas <itabas016@gmail.com>

from nntplib import NNTP
from time import strftime, time, localtime
from email import message_from_string
from urllib.request import urlopen
import textwrap
import re
import json
import datetime, io

day = 24 * 60 * 60 # Number of seconds in one day

def wrap(string, max=70):
    """
    Wraps a string to a maximum line width.
    """

    return '\n'.join(textwrap.wrap(string)) + '\n'

class NewsAgent:
    """
    An object that can distribute news items from news
    sources to news destinations.
    """
    def __init__(self):
        self.sources = []
        self.destinations = []

    def addSource(self, source):
        self.sources.append(source)

    def addDestination(self, dest):
        self.destinations.append(dest)

    def distribute(self):
        """
        Retrieve all news items from all sources, and
        Distribute them to all destinations.
        """
        items = []
        for source in self.sources:
            items.extend(source.getItems())
        for dest in self.destinations:
            dest.receiveItems(items)

class NewsItem:
    """
    A simple news item consisting of a title and a body text.
    """
    def __init__(self, title, body):
        self.title = title
        self.body = body

class NNTPSource:
    """
    A news source that retrieves news items from an NNTP group.
    """
    def __init__(self, servername, group, window):
        self.servername = servername
        self.group = group
        self.window = window

    def getItems(self):

        start = localtime(time() - self.window*day)
        date = strftime('%y%m%d', start)
        hour = strftime('%H%M%S', start)
        full_date = date + time

        server = NNTP(self.servername)

        ids = server.newnews(self.group, datetime.datetime.strptime(full_date, '%y%m%d%H%M%S'))[1]

        for id in ids:
            lines = server.article(id)[3]
            message = message_from_string('\n'.join(lines))

            title = message['subject']
            body = message.get_payload()
            if message.is_multipart():
                body = body[0]

            yield NewsItem(title, body)

        server.quit()

class SimpleWebSource:
    """
    A news source that extracts news items from a Web page using
    regular expressions.
    """
    def __init__(self, url, titlePattern, bodyPattern):
        self.url = url
        self.titlePattern = re.compile(titlePattern)
        self.bodyPattern = re.compile(bodyPattern)

    def getItems(self):
        text = urlopen(self.url).read().decode('utf-8')
        titles = self.titlePattern.findall(text)
        bodies = self.bodyPattern.findall(text)
        for title, body in zip(titles, bodies):
            yield NewsItem(title, wrap(body))

class APIJsonSoucre:
    """
    API response base on json
    """
    def __init__(self, url):
        self.url = url
    
    def getItems(self):
        content = urlopen(self.url).read().decode('utf-8')
        data = json.loads(content)
        status = data['status']
        if status == 'ok':
            articles = data['articles']
            titles = []
            bodies = []
            if len(articles) > 0:
                for article in articles:
                    title = article['title']
                    url = article['url']
                    publish = article['publishedAt']
                    description = article['description']
                    titles.append(title)
                    bodies.append("""
                        <ul style="list-style: none;">
                            <li>Description: %s</li>\n
                            <li>Origin URL: <a href="%s">%s</a></li>\n
                            <li>Published At: %s</li>
                        </ul>
                    """ % (description, url, url, publish))
                for title, body in list(zip(titles, bodies)):
                    yield NewsItem(title, wrap(body))

class StringBuilder:
    _file_str = None

    def __init__(self):
         self._file_str = io.StringIO()

    def Append(self, str):
         self._file_str.write(str)

    def __str__(self):
         return self._file_str.getvalue()

class PlainDestination:
    """
    A news destination that formats all its news items as
    plain text.
    """
    def receiveItems(self, items):
        for item in items:
            print(item.title)
            print('-'*len(item.title))
            print(item.body)


class HTMLDestination:
    """
    A news destination that formats all its news items
    as HTML.
    """
    def __init__(self, filename):
        self.filename = filename

    def receiveItems(self, items):

        out = open(self.filename, 'w')
        print(out, """
        <html>
          <head>
            <title>Today's News</title>
          </head>
          <body>
          <h1>Today's News</h1>
        """)

        print(out, '<ul>')
        id = 0
        for item in items:
            id += 1
            print(out, '  <li><a href="#%i">%s</a></li>' % (id, item.title))
        print(out, '</ul>')

        id = 0
        for item in items:
            id += 1
            print(out, '<h2><a name="%i">%s</a></h2>' % (id, item.title))
            print(out, '<pre>%s</pre>' % item.body)

        print(out, """
          </body>
        </html>
        """)

class HTMLDestination2():
    def __init__(self, filename):
        self.filename = filename

    def receiveItems(self, items):
        
        content = StringBuilder()

        first_part = """
<html>
    <head>
        <title>Top of BBC's News</title>
    </head>
    <body>
        <h1>Top of BBC's News</h1>"""
        content.Append(first_part)
        content.Append("""
        <ul>""")

        for item in items:
            content.Append('<li><b>%s</b>\n%s</li>\n' % (item.title, item.body))
        
        content.Append("""
        </ul>""")

        last_part = """
    </body>
</html>"""
        content.Append(last_part)
        with open(self.filename, 'w', encoding='utf-8') as f:
            f.write(content.__str__())

def runDefaultSetup():
    """
    A default setup of sources and destination. Modify to taste.
    """
    agent = NewsAgent()

    # # A SimpleWebSource that retrieves news from the
    # # BBC news site:
    # bbc_url = 'http://news.bbc.co.uk/text_only.stm'
    # bbc_title = r'(?s)a href="[^"]*">\s*<b>\s*(.*?)\s*</b>'
    # bbc_body = r'(?s)</a>\s*<br />\s*(.*?)\s*<'
    # bbc = SimpleWebSource(bbc_url, bbc_title, bbc_body)

    # agent.addSource(bbc)

    # # How to realize a nntp source, please goto sample
    # # An NNTPSource that retrieves news from comp.lang.python.announce:
    # clpa_server = 'news.foo.bar' # Insert real server name
    # clpa_group = 'comp.lang.python.announce'
    # clpa_window = 1
    # clpa = NNTPSource(clpa_server, clpa_group, clpa_window)

    # agent.addSource(clpa)

    ###
    # Above all source is invalid, currently use news API and
    # JSON format to resolve this. 

    # BBC News API:
    # bbc_news_url = 'https://newsapi.org/v1/articles?source=bbc-news&sortBy=top&apiKey={your-api-key}'
    bbc_news = APIJsonSoucre(bbc_news_url)
    
    agent.addSource(bbc_news)

    # # Add plain text destination and an HTML destination:
    # agent.addDestination(PlainDestination())
    # agent.addDestination(HTMLDestination('news.html'))

    agent.addDestination(HTMLDestination2('news.html'))

    # Distribute the news items:
    agent.distribute()

if __name__ == '__main__':
    runDefaultSetup()