From 19e1ed8b32179e3317c807b3ab0581e3b5fb00a2 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Wed, 4 Jul 2018 00:14:37 +0100 Subject: [PATCH] Implement forum parser to increase accuracy --- app/models.py | 16 ++-- app/tasks/forumtasks.py | 127 +++++++++++++++------------ app/tasks/phpbbparser.py | 70 +++++++++++++++ app/templates/admin/list.html | 3 +- app/templates/macros/topictable.html | 4 +- app/templates/packages/view.html | 2 +- app/views/admin.py | 9 +- app/views/packages/__init__.py | 8 +- app/views/packages/todo.py | 12 +-- app/views/users.py | 6 +- migrations/versions/9fc23495713b_.py | 55 ++++++++++++ 11 files changed, 226 insertions(+), 86 deletions(-) create mode 100644 migrations/versions/9fc23495713b_.py diff --git a/app/models.py b/app/models.py index 5332dbd..b5b4836 100644 --- a/app/models.py +++ b/app/models.py @@ -743,23 +743,25 @@ REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \ "digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \ "imageshack.com", "imgur.com"] -class KrockForumTopic(db.Model): +class ForumTopic(db.Model): topic_id = db.Column(db.Integer, primary_key=True, autoincrement=False) author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) author = db.relationship("User") - ttype = db.Column(db.Integer, nullable=False) + type = db.Column(db.Enum(PackageType), nullable=False) title = db.Column(db.String(200), nullable=False) name = db.Column(db.String(30), nullable=True) link = db.Column(db.String(200), nullable=True) - def getType(self): - if self.ttype == 1 or self.ttype == 2: - return PackageType.MOD - elif self.ttype == 6: - return PackageType.GAME + posts = db.Column(db.Integer, nullable=False) + views = db.Column(db.Integer, nullable=False) + + created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) def getRepoURL(self): + if self.link is None: + return None + for item in REPO_BLACKLIST: if item in self.link: return None diff --git a/app/tasks/forumtasks.py b/app/tasks/forumtasks.py index b2e0ca8..5513fb2 100644 --- a/app/tasks/forumtasks.py +++ b/app/tasks/forumtasks.py @@ -15,12 +15,12 @@ # along with this program. If not, see . -import flask, json +import flask, json, re from flask.ext.sqlalchemy import SQLAlchemy from app import app from app.models import * from app.tasks import celery -from .phpbbparser import getProfile +from .phpbbparser import getProfile, getTopicsFromForum import urllib.request from urllib.parse import urlparse, quote_plus @@ -51,71 +51,88 @@ def checkForumAccount(username, token=None): if needsSaving: db.session.commit() -@celery.task() -def importUsersFromModList(): + +regex_tag = re.compile(r"\[([a-z0-9_]+)\]") +BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"] +def getNameFromTaglist(taglist): + for tag in reversed(regex_tag.findall(taglist)): + if len(tag) < 30 and not tag in BANNED_NAMES and \ + not re.match(r"^[a-z]?[0-9]+$", tag): + return tag + + return None + +regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$") +def parseTitle(title): + m = regex_title.match(title) + if m is None: + print("Invalid title format: " + title) + return title, getNameFromTaglist(title) + else: + return m.group(2).strip(), getNameFromTaglist(m.group(3)) + +def getLinksFromModSearch(): + links = {} + contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8") - list = json.loads(contents) - found = {} - imported = [] + for x in json.loads(contents): + link = x.get("link") + if link is not None: + links[int(x["topicId"])] = link - for user in User.query.all(): - found[user.username] = True - if user.forums_username is not None: - found[user.forums_username] = True - - for x in list: - author = x.get("author") - if author is not None and not author in found: - user = User(author) - user.forums_username = author - imported.append(author) - found[author] = True - db.session.add(user) - - db.session.commit() - for author in found: - checkForumAccount.delay(author, None) - - -BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"] -ALLOWED_TYPES = [1, 2, 6] + return links @celery.task() -def importKrocksModList(): - contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8") - list = json.loads(contents) +def importTopicList(): + links_by_id = getLinksFromModSearch() + + info_by_id = {} + getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.MOD }) + getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME }) + + # Caches username_to_user = {} + topics_by_id = {} + for topic in ForumTopic.query.all(): + topics_by_id[topic.topic_id] = topic - KrockForumTopic.query.delete() + # Create or update + for info in info_by_id.values(): + id = int(info["id"]) - for x in list: - type = int(x["type"]) - if not type in ALLOWED_TYPES: - continue - - username = x["author"] + # Get author + username = info["author"] user = username_to_user.get(username) if user is None: user = User.query.filter_by(forums_username=username).first() - assert(user is not None) + if user is None: + print(username + " not found!") + user = User(username) + user.forums_username = username + db.session.add(user) username_to_user[username] = user - import re - tags = re.findall("\[([a-z0-9_]+)\]", x["title"]) - name = None - for tag in reversed(tags): - if len(tag) < 30 and not tag in BANNED_NAMES and \ - not re.match("^([a-z][0-9]+)$", tag): - name = tag - break + # Get / add row + topic = topics_by_id.get(id) + if topic is None: + topic = ForumTopic() + db.session.add(topic) - topic = KrockForumTopic() - topic.topic_id = x["topicId"] - topic.author_id = user.id - topic.ttype = type - topic.title = x["title"] - topic.name = name - topic.link = x.get("link") - db.session.add(topic) + # Parse title + title, name = parseTitle(info["title"]) + + # Get link + link = links_by_id.get(id) + + # Fill row + topic.topic_id = id + topic.author = user + topic.type = info["type"] + topic.title = title + topic.name = name + topic.link = link + topic.posts = info["posts"] + topic.views = info["views"] + topic.created_at = info["date"] db.session.commit() diff --git a/app/tasks/phpbbparser.py b/app/tasks/phpbbparser.py index d27ccec..9984ad0 100644 --- a/app/tasks/phpbbparser.py +++ b/app/tasks/phpbbparser.py @@ -5,6 +5,7 @@ import urllib, socket from bs4 import * from urllib.parse import urljoin +from datetime import datetime import urllib.request import os.path import time, re @@ -77,3 +78,72 @@ def getProfile(url, username): __extract_properties(profile, soup) return profile + + +regex_id = re.compile(r"^.*t=([0-9]+).*$") + +def parseForumListPage(id, page, out, extra=None): + num_per_page = 30 + start = page*num_per_page+1 + print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page)) + + url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start) + r = urllib.request.urlopen(url).read().decode("utf-8") + soup = BeautifulSoup(r, "html.parser") + + for row in soup.find_all("li", class_="row"): + classes = row.get("class") + if "sticky" in classes or "announce" in classes or "global-announce" in classes: + continue + + topic = row.find("dl") + + # Link info + link = topic.find(class_="topictitle") + id = regex_id.match(link.get("href")).group(1) + title = link.find(text=True) + + # Date + left = topic.find("dt") + date = left.get_text().split("ยป")[1].strip() + date = datetime.strptime(date, "%a %b %d, %Y %H:%M") + author = left.find_all("a")[-1].get_text().strip() + + # Get counts + posts = topic.find(class_="posts").find(text=True) + views = topic.find(class_="views").find(text=True) + + if id in out: + print(" - got {} again, title: {}".format(id, title)) + assert(title == out[id]['title']) + return False + + row = { + "id" : id, + "title" : title, + "author": author, + "posts" : posts, + "views" : views, + "date" : date + } + + if extra is not None: + for key, value in extra.items(): + row[key] = value + + out[id] = row + + return True + +def getTopicsFromForum(id, out={}, extra=None): + print("Fetching all topics from forum {}".format(id)) + page = 0 + while parseForumListPage(id, page, out, extra): + page = page + 1 + + return out + +def dumpTitlesToFile(topics, path): + with open(path, "w") as out_file: + for topic in topics.values(): + out_file.write(topic["title"] + "\n") diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index e5049f9..c565fe0 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -17,8 +17,7 @@