#!/usr/bin/env python

 """

 ch15_popular_links.py

 Build a feed summarizing link popularity found in a set of feeds.

 """

 import sys, re, time, calendar, feedparser

 from scraperlib import FeedEntryDict

 from monitorfeedlib import LogBufferFeed

 from HTMLParser import HTMLParser, HTMLParseError

 FEED_TITLE     = 'Popular Links'

 FEED_TAGLINE   = 'Links found in feed entries ranked by popularity'

 FEED_NAME_FN   = "www/www.decafbad.com/docs/private-feeds/popular-links.%s"

 FEED_DIR       = 'popular-links-feed'

 FEEDS_FN       = 'popular-feed-uris.txt'

 MIN_LINKS      = 3

 MAX_ENTRY_AGE  = 3 * 24 * 60 * 60

 TITLE_TMPL     = """Popular links @ %(time)s (%(link_cnt)s links)"""

 TITLE_TIME_FMT = """%Y-%m-%d %H:%M"""

 CONTENT_TMPL = """

     <div>

         %s

     </div>

 """

 LINK_TMPL = """

     <div style="padding: 10px; margin: 10px; border: 1px solid #aaa;">

         <a style="font-size:1.25em" href="%(link)s">%(link).80s</a><br />

         <i style="font-size:0.75em">(%(link_cnt)s links)</i> 

         <ul>

             %(linkers)s

         </ul>

     </div>

 """

 LINKER_TMPL = """

     <li>

         <a href="%(entry.link)s">%(entry.title)s</a>

         <br />

         <span style="font-size:0.75em">

             [ <a href="%(feed.link)s">%(feed.title)s</a> ]

         </span>

     </li>

 """

 def main():

     """

     Scan all feeds and update the feed with a new link popularity

     report entry.

     """

     # Construct the feed generator.

     f = LogBufferFeed(FEED_DIR)

     f.MAX_AGE = 1 * 24 * 60 * 60 # 1 day

     f.FEED_META['feed.title']   = FEED_TITLE

     f.FEED_META['feed.tagline'] = FEED_TAGLINE

     # Load up the list of feeds.

     feed_uris  = [ x.strip() for x in open(FEEDS_FN,'r').readlines() ]

     # Skim for links from each feed, collect feed and entries in an

     # inverted index using link URLs as top-level keys.

     links = {}

     for feed_uri in feed_uris:

         feed_data = feedparser.parse(feed_uri)

         # Grab the feed metadata from parsed feed.

         feed      = feed_data.feed

         feed_link = feed.get('link', '#')

         # Process all entries for their links...

         for curr_entry in feed_data.entries:

             # HACK: Ignore entries without modification dates.

             # Maybe improve this by stashing seen dates in a DB.

             if curr_entry.get('modified_parsed', None) is None: 

                 continue

             # If the current entry is older than the max allowed age,

             # skip processing it.

             now = time.time()

             entry_time = calendar.timegm(curr_entry.modified_parsed)

             if (now - entry_time) > MAX_ENTRY_AGE:

                 continue

             # Build a LinkSkimmer and feed it all summary and HTML

             # content data from the current entry.  Ignore parse

             # errors in the interest of just grabbing what we can.

             skimmer = LinkSkimmer()

             try:

                 skimmer.feed(curr_entry.get('summary',''))

                 for c in curr_entry.get('content', []): 

                     skimmer.feed(c.value)

             except HTMLParseError:

                 pass

             # Process each link by adding the current feed and entry

             # under the link's key in the inverted index.

             for uri, cnt in skimmer.get_links():

                 if not links.has_key(uri): 

                     links[uri] = {}

                 if not links[uri].has_key(feed_link):

                     links[uri][feed_link] = (feed, curr_entry)

     # Turn the inverted index of links into a list of tuples, sort by

     # popularity of links as measured by number of linked entries.

     links_sorted = links.items()

     links_sorted.sort(lambda a,b: cmp(len(b[1].keys()), len(a[1].keys())))

     # Build the overall entry content from all the links.

     links_out = []

     for x in links_sorted:

         # Get the link and the list of linkers, skip this link if there

         # aren't enough linkers counted.

         link, linkers = x

         if len(linkers) < MIN_LINKS: continue

         # Build the list of linkers for this link by populating the

         # LINKER_TMPL string template.

         linkers_out = []

         for feed, entry in linkers.values():

             linkers_out.append(LINKER_TMPL % {

                 'feed.title'  : feed.get('title', 'untitled'),

                 'feed.link'   : feed.get('link', '#'),

                 'entry.title' : entry.get('title', 'untitled'),

                 'entry.link'  : entry.get('link', '#'),

             })

         # Build the content block for this link by populating the

         # LINK_TMPL string template.

         links_out.append(LINK_TMPL % {

             'link'       : link,

             'link_cnt'   : len(linkers),

             'linkers'    : '\n'.join(linkers_out)

         })

     # Complete building the content for this entry by populating the

     # CONTENT_TMPL string template.

     out = CONTENT_TMPL % '\n'.join(links_out)

     # Construct and append a new entry

     entry = FeedEntryDict({

         'title'   : TITLE_TMPL % {

             'link_cnt' : len(links_out),

             'time'     : time.strftime(TITLE_TIME_FMT)

         },

         'link'    : '',

         'summary' : out

     })

     f.append_entry(entry)

     # Output the current feed entries as both RSS and Atom

     open(FEED_NAME_FN % 'rss', 'w').write(f.scrape_rss())

     open(FEED_NAME_FN % 'atom', 'w').write(f.scrape_atom())

 class LinkSkimmer(HTMLParser):

     """

     Quick and dirty link harvester.

     """

     def reset(self):

         """Reset the parser and the list of links."""

         HTMLParser.reset(self)

         self.links = {}

     def get_links(self):

         """Return the links found as a list of tuples, link and count."""

         return self.links.items()

     def handle_starttag(self, tag, attrs_tup):

         """Harvest href attributes from link tags"""

         attrs = dict(attrs_tup)

         if tag == "a" and attrs.has_key('href'):

             self.links[attrs['href']] = self.links.get(attrs['href'], 0) + 1

 if __name__=='__main__': main()