# ContentDB # Copyright (C) 2018-21 rubenwardy # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import json, re, sys from app.models import * from app.tasks import celery from app.utils import is_username_valid from app.utils.phpbbparser import getProfile, getTopicsFromForum import urllib.request @celery.task() def checkForumAccount(username, forceNoSave=False): print("Checking " + username) try: profile = getProfile("https://forum.minetest.net", username) except OSError: return if profile is None: return user = User.query.filter_by(forums_username=username).first() # Create user needsSaving = False if user is None: user = User(username) user.forums_username = username db.session.add(user) # Get github username github_username = profile.get("github") if github_username is not None and github_username.strip() != "": print("Updated GitHub username for " + user.display_name + " to " + github_username) user.github_username = github_username needsSaving = True pic = profile.avatar if pic and "http" in pic: pic = None needsSaving = needsSaving or pic != user.profile_pic if pic: user.profile_pic = "https://forum.minetest.net/" + pic else: user.profile_pic = None # Save if needsSaving and not forceNoSave: db.session.commit() return needsSaving @celery.task() def checkAllForumAccounts(forceNoSave=False): needsSaving = False query = User.query.filter(User.forums_username.isnot(None)) for user in query.all(): needsSaving = checkForumAccount(user.username) or needsSaving if needsSaving and not forceNoSave: db.session.commit() return needsSaving regex_tag = re.compile(r"\[([a-z0-9_]+)\]") BANNED_NAMES = ["tool", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"] def getNameFromTaglist(taglist): for tag in reversed(regex_tag.findall(taglist)): if len(tag) < 30 and not tag in BANNED_NAMES and \ not re.match(r"^[a-z]?[0-9]+$", tag): return tag return None regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$") def parseTitle(title): m = regex_title.match(title) if m is None: print("Invalid title format: " + title) return title, getNameFromTaglist(title) else: return m.group(2).strip(), getNameFromTaglist(m.group(3)) def getLinksFromModSearch(): links = {} try: contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8") for x in json.loads(contents): try: link = x.get("link") if link is not None: links[int(x["topicId"])] = link except ValueError: pass except urllib.error.URLError: print("Unable to open krocks tool search!") return links return links # @celery.task() # def importTopicList(): # links_by_id = getLinksFromModSearch() # info_by_id = {} # getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.TOOL, 'wip': False }) # getTopicsFromForum(9, out=info_by_id, extra={ 'type': PackageType.TOOL, 'wip': True }) # getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': False }) # getTopicsFromForum(50, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': True }) # # Caches # username_to_user = {} # topics_by_id = {} # for topic in ForumTopic.query.all(): # topics_by_id[topic.topic_id] = topic # def get_or_create_user(username): # user = username_to_user.get(username) # if user: # return user # if not is_username_valid(username): # return None # user = User.query.filter_by(forums_username=username).first() # if user is None: # user = User.query.filter_by(username=username).first() # if user: # return None # user = User(username) # user.forums_username = username # db.session.add(user) # username_to_user[username] = user # return user # # Create or update # for info in info_by_id.values(): # id = int(info["id"]) # # Get author # username = info["author"] # user = get_or_create_user(username) # if user is None: # print("Error! Unable to create user {}".format(username), file=sys.stderr) # continue # # Get / add row # topic = topics_by_id.get(id) # if topic is None: # topic = ForumTopic() # db.session.add(topic) # # Parse title # title, name = parseTitle(info["title"]) # # Get link # link = links_by_id.get(id) # # Fill row # topic.topic_id = int(id) # topic.author = user # topic.type = info["type"] # topic.title = title # topic.name = name # topic.link = link # topic.wip = info["wip"] # topic.posts = int(info["posts"]) # topic.views = int(info["views"]) # topic.created_at = info["date"] # db.session.commit()