Commit 64af851b authored by onny's avatar onny

release

parent ac410af4
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Dependencies: yaourt -S python-requests python-beautifulsoup4 python-pyrss2gen python-bottle python-feedparser python-requests-cache
# base
import logging
# parse
import feedparser
from bs4 import BeautifulSoup
import requests
import requests_cache
import re
# gen feed
import PyRSS2Gen
from time import mktime
from datetime import datetime
# bottle
import bottle
import inspect
import functools
from bottle import run, error, route
# feeds.py plugins
from plugins import ardmediathek
from plugins import ebaykleinanzeigen
from plugins import kitmarkt
from plugins import owncloudapps
from plugins import studentenwerk
#from plugins import artemediathek
from plugins import quoka
from plugins import twitter
from plugins import soundcloud
requests_cache.install_cache(expire_after=1800)
def mail_error(message):
return "Error message"
# base functions
def checkParams(**types):
def decorate(f):
farg, _, _, def_params = inspect.getargspec(f)
if def_params is None: def_params = []
farg = farg[:len(farg) - len(def_params)]
param_info = [(par, ptype, par in farg) for par, ptype in types.items()]
@functools.wraps(f)
def wrapper(*args, **kargs):
getparam = bottle.request.GET.get
for par, ptype, required in param_info:
value = getparam(par)
if not value: # None or empty str
if required:
error = "%s() requires the parameter %s" % (wrapper.__name__, par)
raise TypeError(error)
continue
try:
kargs[par] = ptype(value)
except:
error = "Cannot convert parameter %s to %s" % (par, ptype.__name__)
raise ValueError(error)
return f(*args, **kargs)
return wrapper
return decorate
class MyRSS2(PyRSS2Gen.RSS2):
def publish_extensions(self, handler):
PyRSS2Gen._element(handler, 'icon', 'http://www.kit.edu/img/intern/favicon.ico')
bottle.debug(True)
app = bottle.Bottle()
app.mount('/feeds/ardmediathek',ardmediathek.app)
#app.mount('/feeds/artemediathek',artemediathek.app)
app.mount('/feeds/ebaykleinanzeigen',ebaykleinanzeigen.app)
app.mount('/feeds/kitmarkt',kitmarkt.app)
app.mount('/feeds/owncloudapps',owncloudapps.app)
app.mount('/feeds/studentenwerk',studentenwerk.app)
app.mount('/feeds/quoka',quoka.app)
app.mount('/feeds/twitter',twitter.app)
app.mount('/feeds/soundcloud',soundcloud.app)
app.run(reloader=True, host='0.0.0.0', port=8080)
# if __name__ == '__main__':
# bottle.debug(True)
# bottle.run(reloader=True,host='0.0.0.0', port=8080)
# else:
# os.chdir(os.path.dirname(__file__))
# application = bottle.default_app()
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
{
"folders":
[
{
"follow_symlinks": true,
"path": "."
}
]
}
{
"auto_complete":
{
"selected_items":
[
[
"error",
"error_message"
],
[
"ebay",
"ebaykleinanzeigen"
],
[
"pub",
"published_parsed"
],
[
"ard",
"ardmediathek_getmedia"
]
]
},
"buffers":
[
{
"file": "plugins/ebaykleinanzeigen.py",
"settings":
{
"buffer_size": 3925,
"line_ending": "Unix"
}
},
{
"file": "plugins/quoka.py",
"settings":
{
"buffer_size": 5024,
"line_ending": "Unix"
}
},
{
"file": "feeds.py",
"settings":
{
"buffer_size": 2764,
"line_ending": "Unix"
}
},
{
"file": "plugins/ardmediathek.py",
"settings":
{
"buffer_size": 4416,
"line_ending": "Unix"
}
},
{
"file": "misc.py",
"settings":
{
"buffer_size": 40,
"line_ending": "Unix"
}
}
],
"build_system": "Packages/Python/Python.sublime-build",
"command_palette":
{
"height": 72.0,
"selected_items":
[
[
"package inst",
"Package Control: Install Package"
],
[
"git add",
"Git: Quick Add"
],
[
"git push",
"Git: Push"
],
[
"git commi",
"Git: Commit"
],
[
"git commit",
"Git: Amend Commit"
],
[
"git ini",
"Git: Init"
],
[
"Package Control: Install Package",
"Package Control: Install Package"
]
],
"width": 449.0
},
"console":
{
"height": 243.0
},
"distraction_free":
{
"menu_visible": true,
"show_minimap": false,
"show_open_files": false,
"show_tabs": false,
"side_bar_visible": false,
"status_bar_visible": false
},
"file_history":
[
"/var/www/feeds/testapp.py",
"/var/www/feeds/test.py",
"/var/www/feeds/test_working.py",
"/var/www/feeds/plugins/owncloudapps.py",
"/var/www/feeds/plugins/kitmarkt.py",
"/var/www/feeds/plugins/ardmediathek.py",
"/var/www/feeds/plugins/quoka_test.py",
"/var/www/feeds/plugins/studentenwerk.py",
"/var/www/feeds/plugins/ebaykleinanzeigen.py",
"/var/www/feeds/feeds.py",
"/var/www/feeds/plugins/artemediathek.py",
"/var/www/feeds/plugins/feeds.py",
"/var/www/feeds/ebaykleinanzeigen/ebaykleinanzeigen.py",
"/var/www/feeds/studentenwerk/studentenwerk.py",
"/var/www/feeds/owncloud-apps/owncloud-apps.py",
"/var/www/feeds/kitmarkt/kitmarkt.py",
"/var/www/feeds/ardmediathek/ardmediathek.py",
"/var/www/feeds/owncloud-apps/.owncloud-apps.py.swp",
"/var/www/feeds/ardmediathek/.ardmediathek.py.swp",
"/var/www/feeds/zapp/zapp.py",
"/home/onny/projects/py-pkgcheck/pkgcheck-main.py",
"/home/onny/projects/django-test/tango_with_django_project/templates/rango/about.html",
"/home/onny/projects/django-test/tango_with_django_project/static/test.txt",
"/home/onny/projects/django-test/tango_with_django_project/templates/rango/index.html",
"/home/onny/projects/django-test/tango_with_django_project/rango/admin.py",
"/home/onny/projects/django-test/tango_with_django_project/rango/tests.py"
],
"find":
{
"height": 23.0
},
"find_in_files":
{
"height": 0.0,
"where_history":
[
]
},
"find_state":
{
"case_sensitive": false,
"find_history":
[
"ardmediathek",
"()\n",
"5",
"print",
"re.",
"re",
"count",
"image",
"uri",
"maxent",
"minPrice=",
"app",
"request",
"os",
"app",
" \n",
"datetime",
"datetim",
"sys",
"print",
"description",
"os",
"id",
"publish",
"search",
"STATIC",
"template"
],
"highlight": true,
"in_selection": false,
"preserve_case": false,
"regex": false,
"replace_history":
[
],
"reverse": false,
"show_context": true,
"use_buffer2": true,
"whole_word": false,
"wrap": true
},
"groups":
[
{
"selected": 2,
"sheets":
[
{
"buffer": 0,
"file": "plugins/ebaykleinanzeigen.py",
"settings":
{
"buffer_size": 3925,
"regions":
{
},
"selection":
[
[
3587,
3587
]
],
"settings":
{
"syntax": "Packages/Python/Python.tmLanguage",
"tab_size": 4,
"translate_tabs_to_spaces": true
},
"translation.x": 0.0,
"translation.y": 690.0,
"zoom_level": 1.0
},
"type": "text"
},
{
"buffer": 1,
"file": "plugins/quoka.py",
"settings":
{
"buffer_size": 5024,
"regions":
{
},
"selection":
[
[
4032,
4032
]
],
"settings":
{
"syntax": "Packages/Python/Python.tmLanguage"
},
"translation.x": 0.0,
"translation.y": 797.0,
"zoom_level": 1.0
},
"type": "text"
},
{
"buffer": 2,
"file": "feeds.py",
"settings":
{
"buffer_size": 2764,
"regions":
{
},
"selection":
[
[
765,
765
]
],
"settings":
{
"syntax": "Packages/Python/Python.tmLanguage",
"tab_size": 4,
"translate_tabs_to_spaces": true
},
"translation.x": 0.0,
"translation.y": 0.0,
"zoom_level": 1.0
},
"type": "text"
},
{
"buffer": 3,
"file": "plugins/ardmediathek.py",
"settings":
{
"buffer_size": 4416,
"regions":
{
},
"selection":
[
[
0,
0
]
],
"settings":
{
"syntax": "Packages/Python/Python.tmLanguage"
},
"translation.x": 0.0,
"translation.y": 450.0,
"zoom_level": 1.0
},
"type": "text"
},
{
"buffer": 4,
"file": "misc.py",
"settings":
{
"buffer_size": 40,
"regions":
{
},
"selection":
[
[
38,
38
]
],
"settings":
{
"syntax": "Packages/Python/Python.tmLanguage"
},
"translation.x": 0.0,
"translation.y": 0.0,
"zoom_level": 1.0
},
"type": "text"
}
]
}
],
"incremental_find":
{
"height": 23.0
},
"input":
{
"height": 31.0
},
"layout":
{
"cells":
[
[
0,
0,
1,
1
]
],
"cols":
[
0.0,
1.0
],
"rows":
[
0.0,
1.0
]
},
"menu_visible": true,
"output.exec":
{
"height": 252.0
},
"output.find_results":
{
"height": 0.0
},
"output.git-commit":
{
"height": 100.0
},
"output.git-push":
{
"height": 100.0
},
"replace":
{
"height": 42.0
},
"save_all_on_build": true,
"select_file":
{
"height": 0.0,
"selected_items":
[
],
"width": 0.0
},
"select_project":
{
"height": 500.0,
"selected_items":
[
],
"width": 380.0
},
"show_minimap": false,
"show_open_files": false,
"show_tabs": true,
"side_bar_visible": true,
"side_bar_width": 138.0,
"status_bar_visible": true
}
def mail_error(message):
print("error")
\ No newline at end of file
# base
import logging
# parse
import feedparser
from bs4 import BeautifulSoup
import requests
import requests_cache
import re
import json
# gen feed
import PyRSS2Gen
from time import mktime
from datetime import datetime
# bottle
import bottle
import inspect
import functools
from misc import mail_error
import smtplib
logging.basicConfig(level=logging.DEBUG)
baseuri = "http://www.ardmediathek.de/tv/Weltspiegel/Sendung?rss=true&documentId="
app = bottle.Bottle()
# custom functions
class MyRSS2(PyRSS2Gen.RSS2):
def publish_extensions(self, handler):
PyRSS2Gen._element(handler, 'icon', 'http://www.kit.edu/img/intern/favicon.ico')
def checkParams(**types):
def decorate(f):
farg, _, _, def_params = inspect.getargspec(f)
if def_params is None: def_params = []
farg = farg[:len(farg) - len(def_params)]
param_info = [(par, ptype, par in farg) for par, ptype in types.items()]
@functools.wraps(f)
def wrapper(*args, **kargs):
getparam = bottle.request.GET.get
for par, ptype, required in param_info:
value = getparam(par)
if not value: # None or empty str
if required:
error = "%s() requires the parameter %s" % (wrapper.__name__, par)
raise TypeError(error)
continue
try:
kargs[par] = ptype(value)
except:
error = "Cannot convert parameter %s to %s" % (par, ptype.__name__)
raise ValueError(error)
return f(*args, **kargs)
return wrapper
return decorate
def ardmediathek_getmedia(documentid):
# http://www.ardmediathek.de/tv/Klassiker-der-Weltliteratur/Murasaki-Shikibu/BR-alpha/Video?documentId=15682740&bcastId=14913194
ardmediathek_media_url = re.search(r".*documentId=(\d+).*", documentid)
ardmediathek_media_documentid = ardmediathek_media_url.group(1)
ardmediathek_media = requests.get("http://www.ardmediathek.de/play/media/"+ardmediathek_media_documentid)
ardmediathek_media.encoding = 'utf-8'
ardmediathek_json = json.loads(ardmediathek_media.text)
for ardmediathek_json in ardmediathek_json['_mediaArray']:
if ardmediathek_json['_plugin'] == 1:
ardmediathek_media = {}
ardmediathek_media['url'] = ardmediathek_json['_mediaStreamArray'][-1]['_stream']
ardmediathek_media_header = requests.head(ardmediathek_media['url']).headers
ardmediathek_media['contentlength'] = ardmediathek_media_header['Content-Length']
ardmediathek_media['contenttype'] = ardmediathek_media_header['Content-Type']
return ardmediathek_media
# feed
@app.route('/<id:int>')
#@checkParams(maxentries = int)
def feed(id,maxentries = 1000):
filters = bottle.request.query.filter.split(",")
feed = feedparser.parse(baseuri+str(id))
rss = MyRSS2(
title = feed.feed.title,
link = feed.feed.link,
description = "",
items = [])
if hasattr(feed.feed,'description'):
rss.description = feed.feed.description
if hasattr(feed.feed,'lastBuildDate'):
rss.lastBuildDate = datetime.fromtimestamp(mktime(feed.feed.updated_parsed))
for idx, item in enumerate(feed.entries):
if any(word in item.title for word in filters):
ardmediathek_media = ardmediathek_getmedia(item.link)
if idx >= maxentries:
break
rss.items.append(PyRSS2Gen.RSSItem(
title = item.title,
link = item.link,
description = item.description,
guid = item.id,
pubDate = datetime.fromtimestamp(mktime(item.published_parsed)),
enclosure = PyRSS2Gen.Enclosure(ardmediathek_media['url'],ardmediathek_media['contentlength'],ardmediathek_media['contenttype'])))
return rss.to_xml("utf-8")
# -*- coding: utf-8 -*-
# parse
from bs4 import BeautifulSoup
import requests
import requests_cache
import re
# gen feed
import PyRSS2Gen
from time import mktime
from datetime import datetime, timedelta
# bottle
import bottle
import inspect
import functools
# todo
# - favicon
# general
baseuri = "http://kleinanzeigen.ebay.de/anzeigen/s-suchanfrage.html?keywords=sofa&categoryId=&locationStr=karlsruhe&locationId=&radius=0&sortingField=SORTING_DATE&adType=&posterType=&pageNum=1&action=find&maxPrice=&minPrice="
requests_cache.install_cache(expire_after=1800)
app = bottle.Bottle()
# base functions
def checkParams(**types):
def decorate(f):
farg, _, _, def_params = inspect.getargspec(f)
if def_params is None: def_params = []
farg = farg[:len(farg) - len(def_params)]
param_info = [(par, ptype, par in farg) for par, ptype in types.items()]
@functools.wraps(f)
def wrapper(*args, **kargs):
getparam = bottle.request.GET.get
for par, ptype, required in param_info:
value = getparam(par)
if not value: # None or empty str
if required:
error = "%s() requires the parameter %s" % (wrapper.__name__, par)
raise TypeError(error)
continue
try:
kargs[par] = ptype(value)
except:
error = "Cannot convert parameter %s to %s" % (par, ptype.__name__)
raise ValueError(error)
return f(*args, **kargs)
return wrapper
return decorate
@app.route('/<query>')
@app.route('/<where>/<query>')
@checkParams(maxentries = int, radius = int, minprice = int, maxprice = int)
def feed(query, where = "", maxentries = 1000, radius = 0, minprice = 0, maxprice = 0):
if maxprice == 0:
maxprice = ""
uri = baseuri.replace("sofa",query).replace("karlsruhe",where).replace("minPrice=","minPrice="+str(minprice)).replace("maxPrice=","maxPrice="+str(maxprice)).replace("radius=0","radius="+str(radius))
ebaykleinanzeigen = requests.get(uri)
ebaykleinanzeigen.encoding = 'utf-8'
ebaykleinanzeigen_soup = BeautifulSoup(ebaykleinanzeigen.text)
rss = PyRSS2Gen.RSS2(
title = "Ebay Kleinanzeigen - "+query+ " in "+where,
link = uri,
description = "",
items = [])
if ebaykleinanzeigen.text.find('Es wurden leider keine Anzeigen') == -1:
for idx, item in enumerate(ebaykleinanzeigen_soup.find("ul", {"id": "srchrslt-adtable"}).findAll('li')):
if 'data-imgsrc' in str(item.findAll('section')[0].a):
image = item.findAll('section')[0].a['data-imgsrc']
else:
image = "http://www.designofsignage.com/application/symbol/building/image/600x600/no-photo.jpg"
link = "https://kleinanzeigen.ebay.de/"+item.findAll('section')[0].a['href']
title = item.findAll('section')[1].h2.text
description = item.findAll('section')[1].p.text
price = item.findAll('section')[2].strong.text
location = item.findAll('section')[2].h3.text
date = item.find('section', {'class': 'ad-listitem-addon'}).text.lstrip()
if "Heute" in date or "heute" in date:
date = datetime.strptime(datetime.today().strftime('%d.%m.%Y ')+date, '%d.%m.%Y Heute, %H:%M')
elif "Gestern" in date:
yesterday = datetime.today() - timedelta(1)
date = datetime.strptime(yesterday.strftime('%d.%m.%Y ')+date, '%d.%m.%Y Gestern, %H:%M')
elif date:
date = datetime.strptime(date, '%d.%m.%Y')
if idx >= maxentries:
break
rss.items.append(PyRSS2Gen.RSSItem(
title = title,
link = link,
description = "<table><tr><td><img src='"+image+"' width=200px></td><td>"+"<b>Preis: </b>"+price+"<br><b>Ort: </b>"+location+"<br>"+description+"</td></tr></table>",
guid = PyRSS2Gen.Guid(link),
pubDate = date))
return rss.to_xml("utf-8")
#!/usr/bin/python
# Dependencies: yaourt -S python-requests python-beautifulsoup4 python-pyrss2gen python-bottle python-requests-cache
#base
import logging
from bs4 import BeautifulSoup
import requests