import time import sys import string import MySQLdb REWRITE = "forummap.txt" HOST = ... USER = ... PASS = ... DB = ... conn = MySQLdb.connect(host = HOST, user = USER, passwd = PASS, db = DB) cursor = conn.cursor() FIRST_ID = 256 POST_NORMAL = 0 POST_STICKY = 1 ITEM_UNLOCKED = 0 ITEM_LOCKED = 1 TIME_LARGE = 2147483647 def compress_blanks(items): output = [] for i in range(len(items)): item = items[i].strip(string.whitespace + "\xa0") if item != "" or len(output) and output[-1] != "" and i != len(items)-1: output.append(item) return output def find_first(items): return reduce(lambda x, y: (y.date, y) if y.date < x[0] else x, items, (TIME_LARGE, None))[1] def find_last(items): return reduce(lambda x, y: (y.date, y) if y.date > x[0] else x, items, (0, None))[1] class forum: def __init__(self, id, slug): self.id = id self.slug = slug self.topics = set() self.posts = set() class user: next_id = FIRST_ID taken = set() def __init__(self, login, pword, email, url, date, id = None): if id == None: self.id = user.next_id user.next_id += 1 else: self.id = id while login.lower() in user.taken: login = login + "_" user.taken.add(login.lower()) self.login = login self.pword = pword self.email = email self.url = url self.date = date self.topics = set() self.posts = set() class topic: next_id = FIRST_ID def __init__(self, name, date, forum, user, views, pinned, closed, slug): self.id = topic.next_id topic.next_id += 1 self.name = name self.date = date self.forum = forum self.user = user self.views = views self.pinned = pinned self.closed = closed self.slug = slug self.posts = set() def link(self): if forum_map.has_key(self.forum): if not user_map.has_key(self.user): self.user = None self.forum = forum_map[self.forum] self.forum.topics.add(self) self.user = user_map[self.user] self.user.topics.add(self) return True else: return False class post: next_id = FIRST_ID def __init__(self, content, date, topic, user, forum): self.id = post.next_id post.next_id += 1 self.content = "\n".join(compress_blanks(content.split("\n"))) self.date = date self.topic = topic self.user = user self.forum = forum def link(self): if topic_map.has_key(self.topic) and forum_map.has_key(self.forum): if not user_map.has_key(self.user): self.user = None self.topic = topic_map[self.topic] self.user = user_map[self.user] self.forum = forum_map[self.forum] self.topic.posts.add(self) self.user.posts.add(self) self.forum.posts.add(self) return True else: return False forum_map = { 1 : forum(2, "general-discussion"), 10 : forum(5, "absolute-beginners"), 12 : forum(6, "troubleshooting"), 2 : forum(7, "features-and-requests"), 5 : forum(8, "off-topic"), 4 : forum(9, "projects-and-collaboration-general"), 8 : forum(4, "distributions"), 3 : forum(10, "educational-applications")} user_map = {None : user("anonymous", None, None, None, None, 1)} topic_map = {} post_map = {} def read_from_simple_press(): print "reading users" cursor.execute ("SELECT id, user_login, user_pass, user_email, user_url, unix_timestamp(user_registered) FROM rp_users") pos = 0 while True: row = cursor.fetchone() if not row: print break if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 user_map[row[0]] = user(row[1], row[2], row[3], row[4], row[5]) print "reading topics" cursor.execute ("SELECT topic_id, topic_name, unix_timestamp(topic_date), forum_id, user_id, topic_opened, topic_pinned, topic_status, topic_slug FROM rp_sftopics ORDER BY topic_id") pos = 0 while True: row = cursor.fetchone() if not row: print break if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 topic_map[row[0]] = topic(row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]) print "reading posts" cursor.execute ("SELECT post_id, post_content, unix_timestamp(post_date), topic_id, user_id, forum_id from rp_sfposts") pos = 0 while True: row = cursor.fetchone() if not row: print break if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 post_map[row[0]] = post(row[1], row[2], row[3], row[4], row[5]) print "users:", len(user_map) print "topics:", len(topic_map) print "posts:", len(post_map) print "pruning" # resolve topic links and drop topics with no valid forum for k, v in topic_map.items(): if not v.link(): del topic_map[k] # resolve post links and drop posts with no valid forum or topic for k, v in post_map.items(): if not v.link(): del post_map[k] # drop topics with no posts for k, v in topic_map.items(): if len(v.posts) == 0: del topic_map[k] print "topics:", len(topic_map) print "posts:", len(post_map) def write_to_phpbb(): print "writing users" file = open(REWRITE, "w") pos = 0 for u in user_map.values(): if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 last = find_last(u.posts) if u.id != 1: cursor.execute ("INSERT INTO phpbb_users (user_id, " \ "group_id, " \ "user_regdate, " \ "username, " \ "username_clean, " \ "user_password, " \ "user_email, " \ "user_website, " \ "user_sig, " \ "user_occ, " \ "user_interests, " \ "user_permissions, " \ "user_posts, " \ "user_lastpost_time) " \ "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (u.id, 2, u.date, u.login, u.login.lower(), u.pword, u.email, u.url, "", "", "", "", len(u.posts), last.date if last else 0)) cursor.execute ("INSERT INTO phpbb_user_group (group_id, " \ "user_id, " \ "group_leader, " \ "user_pending) " \ "VALUES (%s, %s, %s, %s)", (2, u.id, 0, 0)) print print "writing topics" pos = 0 for t in topic_map.values(): if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 file.write(t.forum.slug+"/"+t.slug+" http://www.raspberrypi.org/phpBB3/viewtopic.php?f="+str(t.forum.id)+"&t="+str(t.id)+"\n") first = find_first(t.posts) last = find_last(t.posts) cursor.execute ("INSERT INTO phpbb_topics (topic_id, " \ "forum_id, " \ "topic_poster, " \ "topic_title, " \ "topic_time, " \ "topic_replies, " \ "topic_replies_real, " \ "topic_first_post_id, " \ "topic_first_poster_name, " \ "topic_last_post_id, " \ "topic_last_poster_id, " \ "topic_last_poster_name, " \ "topic_last_post_subject, " \ "topic_last_post_time, " \ "topic_views, " \ "topic_type, " \ "topic_status, " \ "topic_time_limit) " \ "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", (t.id, t.forum.id, t.user.id, t.name, t.date, len(t.posts)-1, len(t.posts)-1, first.id, first.user.login, last.id, last.user.id, last.user.login, "Re: "+t.name, last.date, t.views, POST_STICKY if t.pinned else POST_NORMAL, ITEM_LOCKED if t.closed else ITEM_UNLOCKED, TIME_LARGE)) print print "writing posts" pos = 0 for i in post_map: p = post_map[i] if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 file.write(p.forum.slug+"/"+p.topic.slug+"/#p"+str(i)+" http://www.raspberrypi.org/phpBB3/viewtopic.php?f="+str(p.forum.id)+"&t="+str(p.topic.id)+"#p"+str(p.id)+"\n") cursor.execute ("INSERT INTO phpbb_posts (post_id, " \ "topic_id, " \ "forum_id, " \ "poster_id, " \ "post_time, " \ "post_subject, " \ "post_text, " \ "post_checksum) " \ "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", (p.id, p.topic.id, p.forum.id, p.user.id, p.date, "Re: "+p.topic.name, p.content, "1234")) print print "updating forums" pos = 0 for f in forum_map.values(): if pos % 1000 == 0: sys.stdout.write(".") sys.stdout.flush() pos += 1 last = find_last(f.posts) cursor.execute("UPDATE phpbb_forums set forum_posts = %s, " \ "forum_topics = %s, " \ "forum_topics_real = %s, " \ "forum_last_post_id = %s, " \ "forum_last_poster_id = %s, " \ "forum_last_poster_name = %s, " \ "forum_last_post_subject = %s, " \ "forum_last_post_time = %s " \ "WHERE forum_id = %s", (len(f.posts), len(f.topics), len(f.topics), last.id, last.user.id, last.user.login, "Re: "+t.name, last.date, f.id)) print print "updating config" cursor.execute("UPDATE phpbb_config set config_value = %s WHERE config_name = %s", (len(user_map), "num_users")) cursor.execute("UPDATE phpbb_config set config_value = %s WHERE config_name = %s", (len(topic_map), "num_topics")) cursor.execute("UPDATE phpbb_config set config_value = %s WHERE config_name = %s", (len(post_map), "num_posts")) file.close() read_from_simple_press() write_to_phpbb() cursor.close () conn.commit () conn.close ()