Commit 828c4aa1 authored by onny's avatar onny
Browse files

downloading and converting anitya database

parent 966a3888
......@@ -6,10 +6,5 @@
"http://mirror.23media.de/archlinux/extra/os/x86_64/extra.db"
],
"repositories": [ "community", "core", "extra", "aur" ],
"postgres":
{
"user" : "archupstream",
"password" : "archupstream",
"database" : "archupstream"
}
"anitya_db_dump_link": "https://infrastructure.fedoraproject.org/infra/db-dumps/anitya.dump.xz"
}
......@@ -6,6 +6,7 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
ProgressBar, ReverseBar, RotatingMarker, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
import sqlite3
class Database:
......@@ -15,6 +16,62 @@ class Database:
self.pkglist = []
self.packages = []
def convert_upstream_database(self):
try:
os.remove("/tmp/arch-upstream/anitya.sqlite")
except:
pass
conn = sqlite3.connect('/tmp/arch-upstream/anitya.sqlite')
conn_cursor = conn.cursor()
create_table_projects = "CREATE TABLE IF NOT EXISTS projects (id integer NOT NULL, name character varying(200) NOT NULL, latest_version character varying(50));"
create_table_packages = "CREATE TABLE IF NOT EXISTS packages (id integer NOT NULL, distro character varying(200), project_id integer, package_name character varying(200));"
conn_cursor.execute(create_table_projects)
conn_cursor.execute(create_table_packages)
start_reading_packages = 0
start_reading_projects = 0
dump_file = open("/tmp/arch-upstream/anitya.dump",'r')
for line in dump_file:
if re.search("^\\\.",line):
start_reading_packages = 0
start_reading_projects = 0
if start_reading_projects == 1:
table_id = re.findall('^([\d]*)\t', line)[0]
name = re.findall('^[\d]*\t(.*?)\t',line)[0]
latest_version = re.findall('^[\d]*\t.*?\t.*?\t.*?\t.*?\t.*?\t([a-zA-Z0-9_.\-~]*)\t',line)
if len(latest_version):
latest_version = latest_version[0]
else:
latest_version = ""
conn_cursor.execute("INSERT INTO projects (id, name, latest_version) VALUES (?, ?, ?)", (table_id, name, latest_version))
if start_reading_packages == 1:
table_id = re.findall('^([\d]*)\t', line)[0]
distro = re.findall('^[\d]*\t(.*?)\t',line)[0]
project_id = re.findall('^[\d]*\t.*?\t([\d]*)\t',line)
if len(project_id):
project_id = project_id[0]
else:
project_id = ""
package_name = re.findall('^.*?\t.*?\t.*?\t(.*?)\n',line)[0]
conn_cursor.execute("INSERT INTO packages (id, distro, project_id, package_name) VALUES (?, ?, ?, ?)", (table_id, distro, project_id, package_name))
if "COPY packages " in line:
start_reading_packages = 1
if "COPY projects " in line:
start_reading_projects = 1
conn.commit()
dump_file.close()
conn.close()
def parse_database(self, database):
os.chdir(self.temp_directory)
tar = tarfile.open(database, "r:tar")
......
......@@ -6,6 +6,7 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
SimpleProgress, Timer, AdaptiveETA, AbsoluteETA, AdaptiveTransferSpeed
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
from time import sleep
import lzma
class Download:
......@@ -13,8 +14,9 @@ class Download:
self.database_sources = config['database_sources']
self.temp_directory = config['temp_directory']
self.package_database = []
self.anitya_db_dump_link = config['anitya_db_dump_link']
def get_databases(self):
def sync_repos(self):
for idx, link in enumerate(self.database_sources):
......@@ -24,7 +26,7 @@ class Download:
r = requests.get(link, stream=True)
f = open(local_filename, 'wb')
total_length = int(r.headers['Content-Length'].strip())
widgets[0] = 'Downloading databases (' +str(idx+1)+ '/' + str(len(self.database_sources)) +'): '
widgets[0] = 'Downloading repositories (' +str(idx+1)+ '/' + str(len(self.database_sources)) +'): '
bar = ProgressBar(widgets=widgets, maxval=total_length).start()
fbytes = 0
......@@ -40,3 +42,23 @@ class Download:
print("")
return self.package_database
def sync_upstream_database(self):
r = requests.get(self.anitya_db_dump_link, stream=True)
f = open("/tmp/arch-upstream/anitya.dump.xz", 'wb')
total_length = int(r.headers['Content-Length'].strip())
widgets[0] = 'Downloading database : '
bar = ProgressBar(widgets=widgets, maxval=total_length).start()
fbytes = 0
for buf in r.iter_content(1024):
if buf:
f.write(buf)
fbytes += len(buf)
if fbytes < total_length:
bar.update(fbytes)
else:
bar.update(total_length)
f.close()
print("")
from pkg_resources import parse_version
import requests
import json
import sqlite3
from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
FileTransferSpeed, FormatLabel, Percentage, \
......@@ -45,27 +46,38 @@ class Versioncheck:
print("")
return packages_outdated
def get_upstream_version(self, packagename):
response = requests.get('https://release-monitoring.org/api/projects/?pattern='+packagename)
try:
for project in response.json()['projects']:
if project['name'] == packagename:
if project['version']:
return project['version'].replace('.src','')
else:
return "None"
except:
def get_upstream_version(self, package_name, conn):
conn_cursor = conn.cursor()
conn_cursor.execute('select project_id from packages where package_name="'+package_name+'" and distro="Arch Linux"')
packages = conn_cursor.fetchall()
if len(packages):
project_id = str(packages[0][0])
conn_cursor.execute('select latest_version from projects where id="'+project_id+'"')
latest_version = conn_cursor.fetchall()
if len(latest_version):
latest_version = latest_version[0][0]
if latest_version:
return latest_version
else:
return "None"
else:
return "None"
else:
return "None"
def compare_upstream(self):
conn = sqlite3.connect('/tmp/arch-upstream/anitya.sqlite')
widgets[0] = 'Checking upstream versions : '
bar = ProgressBar(widgets=widgets, max_value=len(self.packages))
for idx, package in enumerate(self.packages):
bar.update(idx)
upstreamversion = self.get_upstream_version(package['name'])
upstreamversion = self.get_upstream_version(package['name'],conn)
self.packages[idx]['upstreamversion'] = upstreamversion
if upstreamversion == None:
if upstreamversion == 'None':
self.packages_notlisted.append(self.packages[idx])
else:
if parse_version(upstreamversion) > parse_version(package['version']):
......@@ -75,4 +87,6 @@ class Versioncheck:
bar.update(idx+1)
print("")
conn.close()
return self.packages_notlisted, self.packages_outdated, self.packages_uptodate
......@@ -24,12 +24,15 @@ if __name__ == '__main__':
config = load_config('config/config.json')
# Downloading package database
package_database = Download(config).get_databases()
package_database = Download(config).sync_repos()
Download(config).sync_upstream_database()
packages = Database(package_database, config).load_database()
aur_packages = AUR().scrape_packages()
Database(package_database, config).convert_upstream_database()
#aur_packages = AUR().scrape_packages()
#packages = packages + aur_packages
packages = packages + aur_packages
packages = sorted(packages, key=lambda k: k['name'])
packages_notlisted, packages_outdated, packages_uptodate = Versioncheck(packages).compare_upstream()
......
......@@ -3,5 +3,7 @@ Description=Arch-Upstream
[Service]
Type=simple
User=archupstream
Group=archupstream
WorkingDirectory=/home/onny/www/files/arch-upstream
ExecStart=/home/onny/www/files/arch-upstream/main.py
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment