Commit d1a4a3b9 authored by onny's avatar onny
Browse files

read compressed database dump in memory, unify progressbart output

parent 946cf41a
import os
import tarfile
import re
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
......@@ -7,6 +6,7 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
import sqlite3
import lzma
class Database:
......@@ -17,12 +17,7 @@ class Database:
self.packages = []
def convert_upstream_database(self):
try:
os.remove("/tmp/arch-upstream/anitya.sqlite")
except:
pass
conn = sqlite3.connect('/tmp/arch-upstream/anitya.sqlite')
conn = sqlite3.connect(":memory:")
conn_cursor = conn.cursor()
create_table_projects = "CREATE TABLE IF NOT EXISTS projects (id integer NOT NULL, name character varying(200) NOT NULL, latest_version character varying(50));"
......@@ -34,7 +29,7 @@ class Database:
start_reading_packages = 0
start_reading_projects = 0
dump_file = open("/tmp/arch-upstream/anitya.dump",'r')
dump_file = lzma.open("/tmp/arch-upstream/anitya.dump.xz", "rt")
for line in dump_file:
if re.search("^\\\.",line):
start_reading_packages = 0
......@@ -70,7 +65,7 @@ class Database:
conn.commit()
dump_file.close()
conn.close()
return conn
def parse_database(self, database):
tar = tarfile.open(database, "r:tar")
......@@ -87,7 +82,7 @@ class Database:
return pkglist
def load_database(self):
widgets[0] = 'Loading local database : '
widgets[0] = 'Loading local database : '
bar = ProgressBar(widgets=widgets, max_value=len(self.package_database))
for idx, database in enumerate(self.package_database):
bar.update(idx)
......
......@@ -6,7 +6,6 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
from time import sleep
import lzma
class Download:
......@@ -61,13 +60,4 @@ class Download:
bar.update(total_length)
f.close()
with lzma.open("/tmp/arch-upstream/anitya.dump.xz") as f:
file_content = f.read()
with open("/tmp/arch-upstream/anitya.dump", "wb") as extracted_file:
extracted_file.write(file_content)
f.close()
extracted_file.close()
print("")
......@@ -21,7 +21,7 @@ class Template:
trim_blocks=False)
def write(self):
widgets[0] = 'Writing template : '
widgets[0] = 'Writing template : '
bar = ProgressBar(widgets=widgets, max_value=2)
bar.update(0)
context = { 'packages_outdated' : self.packages_outdated,
......
......@@ -11,11 +11,12 @@ widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_val
class Versioncheck:
def __init__(self, packages):
def __init__(self, packages, sqlite_conn):
self.packages = packages
self.packages_outdated = []
self.packages_uptodate = []
self.packages_notlisted = []
self.sqlite_conn = sqlite_conn
def check_repository_status(self, packagename):
response = requests.get('https://www.archlinux.org/packages/search/json/?name='+packagename)
......@@ -35,7 +36,7 @@ class Versioncheck:
def get_repository_status(self, packages_outdated):
if len(packages_outdated) != 0:
widgets[0] = 'Checking repository status : '
widgets[0] = 'Checking repository status : '
bar = ProgressBar(widgets=widgets, max_value=len(packages_outdated))
for idx, package in enumerate(packages_outdated):
bar.update(idx)
......@@ -46,9 +47,9 @@ class Versioncheck:
print("")
return packages_outdated
def get_upstream_version(self, package_name, conn):
def get_upstream_version(self, package_name):
conn_cursor = conn.cursor()
conn_cursor = self.sqlite_conn.cursor()
conn_cursor.execute('select project_id from packages where package_name="'+package_name+'" and distro="Arch Linux"')
packages = conn_cursor.fetchall()
......@@ -69,13 +70,11 @@ class Versioncheck:
def compare_upstream(self):
conn = sqlite3.connect('/tmp/arch-upstream/anitya.sqlite')
widgets[0] = 'Checking upstream versions : '
widgets[0] = 'Checking upstream versions : '
bar = ProgressBar(widgets=widgets, max_value=len(self.packages))
for idx, package in enumerate(self.packages):
bar.update(idx)
upstreamversion = self.get_upstream_version(package['name'],conn)
upstreamversion = self.get_upstream_version(package['name'])
self.packages[idx]['upstreamversion'] = upstreamversion
if upstreamversion == 'None':
self.packages_notlisted.append(self.packages[idx])
......@@ -87,6 +86,6 @@ class Versioncheck:
bar.update(idx+1)
print("")
conn.close()
self.sqlite_conn.close()
return self.packages_notlisted, self.packages_outdated, self.packages_uptodate
......@@ -28,14 +28,14 @@ if __name__ == '__main__':
Download(config).sync_upstream_database()
packages = Database(package_database, config).load_database()
Database(package_database, config).convert_upstream_database()
sqlite_conn = Database(package_database, config).convert_upstream_database()
#aur_packages = AUR().scrape_packages()
#packages = packages + aur_packages
packages = sorted(packages, key=lambda k: k['name'])
packages_notlisted, packages_outdated, packages_uptodate = Versioncheck(packages).compare_upstream()
packages_outdated = Versioncheck(packages).get_repository_status(packages_outdated)
packages_notlisted, packages_outdated, packages_uptodate = Versioncheck(packages, sqlite_conn).compare_upstream()
packages_outdated = Versioncheck(packages, sqlite_conn).get_repository_status(packages_outdated)
Template(PATH, packages_notlisted, packages_outdated, packages_uptodate).write()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment