Commit ec103426 authored by onny's avatar onny
Browse files

fix urlencode

parent 5afaf077
......@@ -7,11 +7,11 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
import sqlite3
import lzma
from urllib.parse import quote
class Database:
def __init__(self, package_database):
self.package_database = package_database
def __init__(self):
self.pkglist = []
self.packages = []
......@@ -75,12 +75,14 @@ class Database:
content = str(descfile.read())
pkgversion = re.search(r'%VERSION%\\n(?:\d+\:)?(.*?)-',content).group(1)
pkgname = re.search(r'%NAME%\\n(.*?)\\n',content).group(1)
pkglist.append({ 'name' : pkgname, 'version' : pkgversion, 'repository' : '', 'state' : '' })
pkgname_urlencoded = quote(pkgname)
pkglist.append({ 'name' : pkgname, 'name_urlencoded' : pkgname_urlencoded, 'version' : pkgversion, 'repository' : '', 'state' : '' })
descfile.close()
tar.close()
return pkglist
def load_database(self):
def load_database(self, package_database):
self.package_database = package_database
widgets[0] = 'Loading local database : '
bar = ProgressBar(widgets=widgets, max_value=len(self.package_database))
for idx, database in enumerate(self.package_database):
......
......@@ -7,15 +7,19 @@ from progressbar import AnimatedMarker, Bar, BouncingBar, Counter, ETA, \
widgets = ['', Percentage(), ' ', SimpleProgress(format='(%(value)d of %(max_value)d)'), ' ', Bar(), ' ', Timer(), ' ', AdaptiveETA()]
from time import sleep
from config import Config
#from config import Config
class Download:
def __init__(self):
self.database_sources = config['database_sources']
self.temp_directory = config['temp_directory']
self.database_sources = [
"http://mirror.23media.de/archlinux/community/os/x86_64/community.db",
"http://mirror.23media.de/archlinux/core/os/x86_64/core.db",
"http://mirror.23media.de/archlinux/extra/os/x86_64/extra.db"
]
self.temp_directory = "/tmp/arch-upstream" #config['temp_directory']
self.package_database = []
self.anitya_db_dump_link = config['anitya_db_dump_link']
self.anitya_db_dump_link = "https://infrastructure.fedoraproject.org/infra/db-dumps/anitya.dump.xz" #config['anitya_db_dump_link']
def sync_repos(self):
......
......@@ -68,7 +68,6 @@ class Versioncheck:
return "None"
def compare_upstream(self):
widgets[0] = 'Checking upstream versions : '
bar = ProgressBar(widgets=widgets, max_value=len(self.packages))
for idx, package in enumerate(self.packages):
......
......@@ -19,17 +19,17 @@ if __name__ == '__main__':
packages = Database().load_database(package_database)
# Convert upstream database to sqlite
# Returns sqlite connection
sqlite_conn = Database().convert_upstream_database(database_dump)
sqlite_conn = Database().convert_upstream_database()
# Scrape AUR packages
aur_packages = AUR().scrape_packages()
packages = packages + aur_packages
#packages = packages + aur_packages
packages = sorted(packages, key=lambda k: k['name'])
# Compare package versions
packages = Versioncheck().compare_versions(packages, sqlite_conn)
packages = Versioncheck(packages, sqlite_conn).compare_upstream()
# Get repository state (flagged state)
packages = Versioncheck().get_repository_status(packages)
packages = Versioncheck(packages, sqlite_conn).get_repository_status(packages)
# Write HTML pages
Template(packages).write()
Template("/home/onny/projects/arch-upstream", packages).write()
......@@ -48,12 +48,12 @@
{% if package['state'] == "notlisted" and package['repository'] != 'AUR' %}
<tr>
{% if package['repository'] == "AUR" %}
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% else %}
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% endif %}
<td>{{ package['version'] }}</td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name'] }}>{{ package['upstreamversion'] }}</a></td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name_urlencoded'] }}>{{ package['upstreamversion'] }}</a></td>
{% if package['status'] == "Unknown" %}
<td><span class="red">{{ package['status'] }}</span></td>
{% endif %}
......
......@@ -48,12 +48,12 @@
{% if package['state'] == "notlisted" and package['repository'] == 'AUR' %}
<tr>
{% if package['repository'] == "AUR" %}
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% else %}
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% endif %}
<td>{{ package['version'] }}</td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name'] }}>{{ package['upstreamversion'] }}</a></td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name_urlencoded'] }}>{{ package['upstreamversion'] }}</a></td>
{% if package['status'] == "Unknown" %}
<td><span class="red">{{ package['status'] }}</span></td>
{% endif %}
......
......@@ -48,12 +48,12 @@
{% if package['state'] == "outdated" and package['repository'] != 'AUR' %}
<tr>
{% if package['repository'] == "AUR" %}
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% else %}
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% endif %}
<td>{{ package['version'] }}</td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name'] }}>{{ package['upstreamversion'] }}</a></td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name_urlencoded'] }}>{{ package['upstreamversion'] }}</a></td>
{% if package['status'] == "Unknown" %}
<td><span class="red">{{ package['status'] }}</span></td>
{% endif %}
......
......@@ -48,12 +48,12 @@
{% if package['state'] == "outdated" and package['repository'] == 'AUR' %}
<tr>
{% if package['repository'] == "AUR" %}
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://aur.archlinux.org/packages/{{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% else %}
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name'] }}>{{ package['name'] }}</a></td>
<td><a target=new href=https://www.archlinux.org/packages/?name={{ package['name_urlencoded'] }}>{{ package['name'] }}</a></td>
{% endif %}
<td>{{ package['version'] }}</td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name'] }}>{{ package['upstreamversion'] }}</a></td>
<td><a target=new href=https://release-monitoring.org/projects/search/?pattern={{ package['name_urlencoded'] }}>{{ package['upstreamversion'] }}</a></td>
{% if package['status'] == "Unknown" %}
<td><span class="red">{{ package['status'] }}</span></td>
{% endif %}
......
[Unit]
Description=Arch-Upstream
[Service]
Type=simple
User=onny
Group=onny
PrivateDevices=yes
PrivateTmp=yes
ProtectSystem=full
CapabilityBoundingSet=
ReadWriteDirectories=/usr/share/webapps/arch-upstream
ProtectHome=yes
NoNewPrivileges=yes
WorkingDirectory=/usr/share/webapps/arch-upstream
ExecStart=/usr/share/webapps/arch-upstream/main.py
[Unit]
Description=Run arch-upstream every 12 hours
[Timer]
# Time to wait after booting before we run first time
OnBootSec=10min
# Time between running each consecutive time
OnUnitActiveSec=12h
Unit=arch-upstream.service
[Install]
WantedBy=multi-user.target
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment