Commit 7aa2e658 authored by onny's avatar onny
Browse files

cleaning up and restructuring the code

parent 92ec4fd6
{
"temp_directory": "/tmp/arch-upstream",
"database_sources": [
"http://mirror.23media.de/archlinux/core/os/x86_64/core.db",
"http://mirror.23media.de/archlinux/extra/os/x86_64/extra.db",
"http://mirror.23media.de/archlinux/community/os/x86_64/community.db"
"http://mirror.23media.de/archlinux/core/os/x86_64/core.db"
]
}
import os
import tarfile
import re
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
class Database:
def __init__(self, package_database, config):
self.package_database = package_database
self.temp_directory = config['temp_directory']
self.pkglist = pkglist = []
self.packages = []
def parse_database(self, database):
os.chdir(self.temp_directory)
tar = tarfile.open(database, "r:tar")
tar.extractall()
for entry in tar.getmembers():
if entry.isfile() and "desc" in entry.name:
descfile = tar.extractfile(entry)
content = str(descfile.read())
pkgversion = re.search(r'%VERSION%\\n(?:\d+\:)?(.*?)-',content).group(1)
pkgname = re.search(r'%NAME%\\n(.*?)\\n',content).group(1)
self.pkglist.append({ 'name' : pkgname, 'version' : pkgversion })
descfile.close()
tar.close()
return self.pkglist
def load_database(self):
widgets[0] = 'Loading local database : '
bar = ProgressBar(widgets=widgets, max_value=len(self.package_database))
for idx, database in enumerate(self.package_database):
bar.update(idx)
self.packages = sum([self.packages, self.parse_database(database)], [])
bar.update(idx+1)
print("")
print(self.packages)
return self.packages
import requests
import os
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
class Download:
def __init__(self, config):
self.database_sources = config['database_sources']
self.temp_directory = config['temp_directory']
self.package_database = []
def get_databases(self):
for idx, link in enumerate(self.database_sources):
local_filename = self.temp_directory + "/" + os.path.split(link)[1]
self.package_database.append(local_filename)
r = requests.get(link, stream=True)
f = open(local_filename, 'wb')
file_size = int(r.headers['Content-Length'])
chunk = 1
num_bars = file_size / chunk
widgets[0] = 'Downloading databases (' +str(idx+1)+ '/' + str(len(self.database_sources)) +'): '
bar = ProgressBar(widgets=widgets, maxval=num_bars).start()
i = 0
for chunk in r.iter_content():
f.write(chunk)
bar.update(i)
i+=1
f.close()
print("")
return self.package_database
from jinja2 import Environment, FileSystemLoader
import datetime
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
import os
class Template:
def __init__(self, PATH, packages_notlisted, packages_outdated, packages_uptodate):
self.packages_outdated = packages_outdated
self.packages_uptodate = packages_uptodate
self.packages_notlisted = packages_notlisted
self.datenow = datetime.datetime.now().strftime("%y/%m/%d %H:%M")
self.PATH = PATH
self.template_environement = Environment(
autoescape=False,
loader=FileSystemLoader(os.path.join(self.PATH, 'templates')),
trim_blocks=False)
def write(self):
widgets[0] = 'Writing template : '
bar = ProgressBar(widgets=widgets, max_value=2)
bar.update(0)
context = { 'packages_outdated' : self.packages_outdated,
'packages_uptodate' : self.packages_uptodate,
'packages_notlisted' : self.packages_notlisted,
'date' : self.datenow}
html = self.template_environement.get_template('upstream.html').render(context)
os.chdir(self.PATH)
with open('upstream.html', 'w') as f:
f.write(html)
bar.update(1)
html = self.template_environement.get_template('notlisted.html').render(context)
with open('notlisted.html', 'w') as f:
f.write(html)
bar.update(2)
from pkg_resources import parse_version
import requests
import json
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
class Versioncheck:
def __init__(self, packages):
self.packages = packages
self.packages_outdated = []
self.packages_uptodate = []
self.packages_notlisted = []
def check_repository_status(packagename):
response = requests.get('https://www.archlinux.org/packages/search/json/?name='+packagename)
try:
for package in response.json()['results']:
if package['pkgname'] == packagename:
if "testing" in package['repo']:
return "Testing"
for package in response.json()['results']:
if package['pkgname'] == packagename:
if package['flag_date'] == None:
return "Unknown"
else:
return "Flagged"
except:
return "Unknown"
def get_repository_status(self, packages_outdated):
if len(packages_outdated) != 0:
widgets[0] = 'Checking repository status : '
bar = ProgressBar(widgets=widgets, max_value=len(packages_outdated))
for idx, package in enumerate(packages_outdated):
bar.update(idx)
status = self.check_repository_status(package['name'])
self.packages_outdated[idx]['status'] = status
bar.update(idx+1)
print("")
return self.packages_outdated
def get_upstream_version(self, packagename):
response = requests.get('https://release-monitoring.org/api/projects/?pattern='+packagename)
try:
for project in response.json()['projects']:
if project['name'] == packagename:
if project['version']:
return project['version'].replace('.src','')
else:
return "None"
except:
return "None"
def compare_upstream(self):
widgets[0] = 'Checking upstream versions : '
bar = ProgressBar(widgets=widgets, max_value=len(self.packages))
for idx, package in enumerate(self.packages):
bar.update(idx)
upstreamversion = self.get_upstream_version(package['name'])
self.packages[idx]['upstreamversion'] = upstreamversion
if upstreamversion == None:
self.packages_notlisted.append(self.packages[idx])
else:
if parse_version(upstreamversion) > parse_version(package['version']):
self.packages_outdated.append(self.packages[idx])
else:
self.packages_uptodate.append(self.packages[idx])
bar.update(idx+1)
print("")
return self.packages_notlisted, self.packages_outdated, self.packages_uptodate
#!/usr/bin/python
import tarfile
import re
import os
from jinja2 import Environment, FileSystemLoader
import requests
import time
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
from pkg_resources import parse_version
import itertools
from clint.textui import progress
import json
import datetime
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
from lib.download import Download
from lib.database import Database
from lib.versioncheck import Versioncheck
from lib.template import Template
PATH = os.path.dirname(os.path.abspath(__file__))
TEMPLATE_ENVIRONMENT = Environment(
autoescape=False,
loader=FileSystemLoader(os.path.join(PATH, 'templates')),
trim_blocks=False)
def load_config(config):
data = []
......@@ -31,123 +17,13 @@ def load_config(config):
os.makedirs(data['temp_directory'], exist_ok=True)
return data
def load_database(package_database):
os.chdir("/tmp/archlinux-anitya")
pkglist = []
tar = tarfile.open(database, "r:tar")
tar.extractall()
for entry in tar.getmembers():
if entry.isfile() and "desc" in entry.name:
descfile = tar.extractfile(entry)
content = str(descfile.read())
pkgversion = re.search(r'%VERSION%\\n(?:\d+\:)?(.*?)-',content).group(1)
pkgname = re.search(r'%NAME%\\n(.*?)\\n',content).group(1)
pkglist.append({ 'name' : pkgname, 'version' : pkgversion })
descfile.close()
tar.close()
return pkglist
if __name__ == '__main__':
def get_upstream_version(packagename):
response = requests.get('https://release-monitoring.org/api/projects/?pattern='+packagename)
try:
for project in response.json()['projects']:
if project['name'] == packagename:
if project['version']:
return project['version'].replace('.src','')
else:
return "None"
except:
return "None"
config = load_config('config/config.json')
def get_repo_status(packagename):
response = requests.get('https://www.archlinux.org/packages/search/json/?name='+packagename)
try:
for package in response.json()['results']:
if package['pkgname'] == packagename:
if "testing" in package['repo']:
return "Testing"
for package in response.json()['results']:
if package['pkgname'] == packagename:
if package['flag_date'] == None:
return "Unknown"
else:
return "Flagged"
except:
return "Unknown"
package_databases = []
config = load_config('config/config.json')
for idx, link in enumerate(config['database_sources']):
local_filename = config['temp_directory'] + "/" + os.path.split(link)[1]
package_databases.append(local_filename)
r = requests.get(link, stream=True)
f = open(local_filename, 'wb')
file_size = int(r.headers['Content-Length'])
chunk = 1
num_bars = file_size / chunk
widgets[0] = 'Downloading databases (' +str(idx+1)+ '/' + str(len(config['database_sources'])) +'): '
bar = ProgressBar(widgets=widgets, maxval=num_bars).start()
i = 0
for chunk in r.iter_content():
f.write(chunk)
bar.update(i)
i+=1
f.close()
print("")
widgets[0] = 'Loading local database : '
bar = ProgressBar(widgets=widgets, max_value=len(package_databases))
packages = []
for idx, database in enumerate(package_databases):
bar.update(idx)
packages = sum([packages, load_database(database)], [])
bar.update(idx+1)
print("")
packages_outdated = []
packages_uptodate = []
packages_notlisted = []
widgets[0] = 'Checking upstream versions : '
bar = ProgressBar(widgets=widgets, max_value=len(packages))
for idx, package in enumerate(packages):
bar.update(idx)
upstreamversion = get_upstream_version(package['name'])
packages[idx]['upstreamversion'] = upstreamversion
if upstreamversion == None:
packages_notlisted.append(packages[idx])
else:
if parse_version(upstreamversion) > parse_version(package['version']):
packages_outdated.append(packages[idx])
else:
packages_uptodate.append(packages[idx])
bar.update(idx+1)
print("")
if len(packages_outdated) != 0:
widgets[0] = 'Checking repository status : '
bar = ProgressBar(widgets=widgets, max_value=len(packages_outdated))
for idx, package in enumerate(packages_outdated):
bar.update(idx)
status = get_repo_status(package['name'])
packages_outdated[idx]['status'] = status
bar.update(idx+1)
print("")
widgets[0] = 'Writing template : '
bar = ProgressBar(widgets=widgets, max_value=2)
bar.update(0)
context = { 'packages_outdated' : packages_outdated,
'packages_uptodate' : packages_uptodate,
'packages_notlisted' : packages_notlisted,
'date' : datetime.datetime.now().strftime("%y/%m/%d %H:%M")}
html =TEMPLATE_ENVIRONMENT.get_template('upstream.html').render(context)
os.chdir(PATH)
with open('upstream.html', 'w') as f:
f.write(html)
bar.update(1)
html =TEMPLATE_ENVIRONMENT.get_template('notlisted.html').render(context)
with open('notlisted.html', 'w') as f:
f.write(html)
bar.update(2)
package_database = Download(config).get_databases()
packages = Database(package_database, config).load_database()
packages = packages[0:20]
packages_notlisted, packages_outdated, packages_uptodate = Versioncheck(packages).compare_upstream()
packages_outdated = Versioncheck(packages).get_repository_status(packages_outdated)
Template(PATH, packages_notlisted, packages_outdated, packages_uptodate).write()
......@@ -65,8 +65,8 @@
</table>
</div>
<i>{{ packages_notlisted|count }} packages outdated. Page last updated: {{ date }} UTC</i>
<i>{{ packages_notlisted|count }} packages not listed. Page last updated: {{ date }} UTC</i>
<br><br>
</div>
</body></html>
......@@ -74,7 +74,7 @@
</div>
<i>{{ packages_outdated|count }} packages outdated. Page last updated: {{ date }} UTC</i>
<br><br>
</div>
</body></html>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment