Cheers, Irelativism here
Here I send a simple patch for the wip branch of the wikimigration scripts in
the infrastructure repository
This small changes hopefully solve the errors we are getting with permissions,
so repo can be created.
P.S. please ignore first email regarding this I sended patch as atachement
instead of inline like here, sorry for the mistake, this should correct it.
Carpe Diem, Alexis
DiEM25.org Exarcheia/Tech.sov DSC
Sent with [ProtonMail](https://protonmail.com/) Secure Email.
From 6e1d1a6ac587034cd580af8b07c4caff4e7f1fc4 Mon Sep 17 00:00:00 2001
From: _I3^RELATIVISM <sk-ale...@protonmail.ch>
Date: Tue, 18 Jan 2022 11:13:55 +0000
Subject: [PATCH] attemped fix permissions errors
---
redmine2git.py | 197 ++++++++++++++++++++++++++++++++-----------------
1 file changed, 128 insertions(+), 69 deletions(-)
diff --git a/redmine2git.py b/redmine2git.py
index a65c8b0..3f1b446 100755
--- a/redmine2git.py
+++ b/redmine2git.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
# Copyright (C) 2021 Denis 'GNUtoo' Carikli <gnu...@cyberdimension.org>
+# Copyright (C) 2021 _I3^IRELATIVISM <sk-ale...@protonmail.ch>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -26,26 +27,27 @@
# for this program and the licenses of the other code concerned.
import os
+import re
import sys
import sh
from types import MethodType
-from redminelib import Redmine
+import redminelib
# We want to order all the wiki changes by date to get a linear git history
def usage(progname):
- print("Usage: {0} <URL> <PROJECT> <textile|markdown> <DIRECTORY>".format(
+ print('Usage: {0} <URL> <PROJECT> <textile|markdown> <DIRECTORY>'.format(
progname))
sys.exit(1)
def write_page(extension, output_directory, page):
file_path = None
- if "parent" in dir(page):
+ if 'parent' in dir(page):
parent_dir_path = output_directory + os.sep + page.parent.title
if not os.path.exists(parent_dir_path):
- os.mkdir(parent_dir_path)
+ os.mkdir(parent_dir_path, mode=0o777, exist_ok=False)
file_path = parent_dir_path + os.sep + page.title + extension
else:
file_path = output_directory + os.sep + page.title + extension
@@ -54,11 +56,34 @@ def write_page(extension, output_directory, page):
f.write(page.text)
f.close()
-def git_commit(extension, output_directory, page):
+ return [file_path]
+
+def write_attachments(output_directory, page):
+ results = []
+ if not page.attachments:
+ return results
+
+ parent_dir_path = None
+ if 'parent' in dir(page):
+ parent_dir_path = output_directory + os.sep \
+ + page.parent.title + page.title
+ if not os.path.exists(parent_dir_path):
+ os.mkdir(parent_dir_path, mode = 0o777)
+ else:
+ parent_dir_path = output_directory + os.sep + page.title
+
+ for attachement in page.attachments:
+ # TODO: print attachement.filename
+ attachement.download(savepath=parent_dir_path)
+ results.append(parent_dir_path + os.path + attachement.filename)
+
+ return results
+
+def git_commit(extension, output_directory, page, file_paths):
# +=============+==================================+========+
# | Page fields | Conversion in git | Status |
# +=============+==================================+========+
- # | attachments | File | |
+ # | attachments | File | DONE |
# +-------------+----------------------------------+--------+
# | author | - Commit author | DONE |
# | | - Commiter | |
@@ -79,85 +104,119 @@ def git_commit(extension, output_directory, page):
# | version | Add in commit message | DONE |
# | | Redmine-page-version: <version> | DONE |
# +-------------+----------------------------------+--------+
- file_path = None
-
- if "parent" in dir(page):
- file_path = page.parent.title + os.sep + page.title + extension
- else:
- file_path = page.title + extension
- sh.git("-C", output_directory, "add", file_path)
+ regex = re.compile('^' + re.escape(output_directory) + os.sep)
+ for file_path in file_paths:
+ path = re.sub(regex, '', file_path)
+ sh.git('-C', output_directory, 'add', file_path)
- commit_author="{author} <{author}@redmine.replicant.us>".format(
- author=page.author)
+ commit_author='{author} <{author}@redmine.replicant.us>'.format(
+ author=redmine_instance.users.get(page.author.id))
commit_date = page.updated_on
if len(page.comments) > 1:
- commit_message_sumarry="{}: {}\n".format(page.title, page.comments)
+ commit_message_sumarry='{}: {}\n'.format(page.title, page.comments)
else:
- commit_message_sumarry="{}: page version {}\n".format(page.title,
- page.version)
+ commit_message_sumarry='{}: page version {}\n'.format(page.title,
+ page.version)
commit_message = commit_message_sumarry
- commit_message += "\n"
- commit_message += "Redmine-page-version: {}".format(page.version)
+ commit_message += '\n'
+ commit_message += 'Redmine-page-version: {}'.format(page.version)
# Print commit preview
- print("-" * 80)
- print("Author: {}".format(commit_author))
- print("Date: {}".format(commit_date))
+ print('-' * 80)
+ print('Author: {}'.format(commit_author))
+ print('Date: {}'.format(commit_date))
+
# Commit message
- for line in commit_message.split("\n"):
- print(" " + line)
+ for line in commit_message.split('\n'):
+ print(' ' + line)
+
+ if len(file_paths) > 0:
+ print('')
+ for file_path in file_paths:
+ print('# Files to be committed:')
+ print('# File: {}'.format(file_path))
+ print('')
- print("-" * 80)
+ print('-' * 80)
# Git commands
command_args = []
command_args.append(
- "--author=\"{author} <{author}@redmine.replicant.us>\"".format(
+ '--author="{author} <{author}@redmine.replicant.us>"'.format(
author=commit_author))
- command_args.append("--message={}".format(commit_message))
- command_args.append("--cleanup=verbatim")
+ command_args.append('--message={}'.format(commit_message))
+ command_args.append('--cleanup=verbatim')
command_args.append('--date="{date}"'.format(date=commit_date))
- print (" ".join(["git", "-C", output_directory, "commit" ] + command_args))
+ print (' '.join(['git', '-C', output_directory, 'commit' ] + command_args))
+
+ sh.git('-C', output_directory, 'commit', *command_args)
+
+
+def get_versioned_page(redmine_instance, redmine_project,
+ page, page_version, nr_versioned_page, nr_versioned_pages,
+ retries=10):
+ while retries > 0:
+ try:
+ versioned_page = redmine_instance.wiki_page.get(
+ page.title,
+ project_id=redmine_project.identifier,
+ version=page_version)
+
+ print(' [{}/{}]: {} @ version {}'.format(nr_versioned_page,
+ nr_versioned_pages,
+ versioned_page.title,
+ versioned_page.version))
+ return versioned_page
+ except redminelib.exceptions.ServerError as e:
+ # When this error occurs we have the folloing information when
+ # printing it with print(e):
+ # Redmine returned internal error, check Redmine logs for details
+ # Since we don't self host our Redmine instance yet, we probably
+ # don't have access to the redmine logs, however we can retry
+ print(' [{}/{}]: Server error: retrying {} @ version {}'.format(
+ nr_versioned_page,
+ nr_versioned_pages,
+ page.title,
+ page_version))
+ finally:
+ retries -= 1
+
+ return None
- sh.git("-C", output_directory, "commit", *command_args)
def sort_pages_by_date(redmine_instance, redmine_project):
all_pages = []
nr_pages = len(redmine_project.wiki_pages)
- nr_versions = 0
- print("Found {} pages.".format(nr_pages))
+ nr_versioned_pages = 0
+ nr_versioned_page = 0
+ print('Found {} pages.'.format(nr_pages))
for page in redmine_project.wiki_pages:
- nr_versions += page.version
+ nr_versioned_pages += page.version
+
+ print('Total: {} version of {} pages.'.format(nr_versioned_pages, nr_pages))
+ print('')
+ print('Sorting pages by date:')
- print("Total: {} version of {} pages.".format(nr_versions, nr_pages))
- print("")
- print("Sorting pages by date:")
- version_nr = 0
for page in redmine_project.wiki_pages:
- for version in range(1, page.version +1):
- version_nr += 1
- page_version = redmine_instance.wiki_page.get(
- page.title,
- project_id=redmine_project.identifier,
- version=version)
- print(" [{}/{}]: {} @ version {}".format(version_nr,
- nr_versions,
- page_version.title,
- page_version.version))
- all_pages.append(page_version)
+ for page_version in range(1, page.version +1):
+ nr_versioned_page += 1
+ versioned_page = get_versioned_page(redmine_instance, redmine_project,
+ page, page_version,
+ nr_versioned_page, nr_versioned_pages)
+ all_pages.append(versioned_page)
# This should sort by modification date
- all_pages.sort(key=lambda item: item.updated_on)
+ #all_pages.sort(key=lambda item: item.updated_on)
- return all_pages
+ #return all_pages
def main():
if len(sys.argv) != 5:
@@ -168,32 +227,32 @@ def main():
redmine_wiki_format = sys.argv[3]
output_directory = sys.argv[4]
- extension = ""
+ extension = ''
- if redmine_wiki_format not in ["textile", "markdown"]:
+ if redmine_wiki_format not in ['textile', 'markdown']:
usage(sys.argv[0])
- elif redmine_wiki_format == "markdown":
- extension = ".md"
- elif redmine_wiki_format == "textile":
- extension = ".textile"
+ elif redmine_wiki_format == 'markdown':
+ extension = '.md'
+ elif redmine_wiki_format == 'textile':
+ extension = '.textile'
- redmine_instance = Redmine(redmine_instance_url)
+ redmine_instance = redminelib.Redmine(redmine_instance_url)
redmine_project = redmine_instance.project.get(redmine_project_name)
pages = sort_pages_by_date(redmine_instance, redmine_project)
if not os.path.exists(output_directory):
- os.mkdir(output_directory)
+ os.mkdir(output_directory, mode = 0o777)
- if not os.path.exists(output_directory + os.sep + ".git"):
- sh.git("-C", output_directory, "init")
+ if not os.path.exists(output_directory + os.sep + '.git'):
+ sh.git('-C', output_directory, 'init')
for page in pages:
- # DEBUG
- print(page.title, page.updated_on)
- # END DEBUG
- write_page(extension, output_directory, page)
- git_commit(extension, output_directory, page)
-
-if __name__ == "__main__":
- main()
+ files = []
+ files += write_page(extension, output_directory, page)
+ files += write_attachments(output_directory, page)
+ git_commit(extension, output_directory, page, files)
+
+if __name__ == '__main__':
+ main()
+
--
2.20.1_______________________________________________
Replicant mailing list
Replicant@osuosl.org
https://lists.osuosl.org/mailman/listinfo/replicant
