import time;
import urllib;
import re;
import os;
from shutil import move;
class PollItem:
def __init__(self, url, period):
self.url=url;
self.period=period;
self.remaining=0;
def elapse(self, time):
self.remaining -= time;
def reset(self):
self.remaining=self.period;
def __cmp__(self, item):
if self.remaining < item.remaining:
return -1;
if self.remaining > item.remaining:
return 1;
return 0;
class FeedWatcher:
def __init__(self, feeds, dest, dirapi):
self.feeds=feeds
self.dest=dest
self.dirapi="%s/feeds/download"%dirapi
self.pollitems=[]
file = open(feeds, 'r')
for line in file:
if line.startswith("FEED"):
(type, url, period)=line.split(' ')
period=float(period)
self.pollitems.append(PollItem(url, period))
while(1):
while(self.pollitems[0].remaining <= 0):
print "Checking",self.pollitems[0].url
self.check(self.pollitems[0].url)
self.pollitems[0].reset()
self.pollitems.sort()
next=float(self.pollitems[0].remaining)
print "Next check in",next
time.sleep(next)
for item in self.pollitems:
item.elapse(next);
def check(self, url):
site=urllib.urlopen(url);
outurl=re.sub("/",".",url)
outurl=outurl[7:]
destfilename="%s/%s"%(self.dest,outurl)
difffilename="%s/%s-diff"%(self.dirapi,outurl)
tempfilename="%s/current"%(self.dest)
#If the file exists, create an empty file
#System works by always sending a diff, so
#The initial condition is a diff against empty file
if not os.path.isfile(destfilename):
outfile=open(destfilename,'w');
outfile.close();
#The file current is a temp file that holds the
#latest fetched feed for comparison.
outfile=open(tempfilename,'w')
for line in site:
outfile.write(line)
outfile.close()
#Now, do the diff, write to difffilename
diffcmd="diff -u %s %s"%(destfilename,tempfilename)
print diffcmd
diff=os.popen(diffcmd).readlines()
if len(diff)>0:
outfile=open(difffilename,'w')
outfile.write("FEED %s\n"%outurl)
for line in diff:
outfile.write(line);
outfile.close()
#Finally, copy the whole thing in case we need to send it all
#over, later.
move(tempfilename, destfilename)
if __name__=="__main__":
watcher = FeedWatcher("src/feeds.conf","feeds")