From 59589726736740ea5d1c0d95a2a7e1e60b40af39 Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Mon, 20 Dec 2021 12:22:26 +0300 Subject: [PATCH 01/19] Support using PostgreSQL database too Use pg gem and use query syntax compatible with it. --- Gemfile | 1 + Gemfile.lock | 13 +++++++++++++ trac-hub | 5 +++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index fb2b575..995a9ab 100644 --- a/Gemfile +++ b/Gemfile @@ -2,6 +2,7 @@ source 'https://rubygems.org' gem 'sequel' gem 'sqlite3' gem 'mysql2' +gem 'pg' gem 'rest-client' diff --git a/Gemfile.lock b/Gemfile.lock index 43a91d4..2ec91b1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,28 +3,41 @@ GEM specs: domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) + ffi (1.15.4-x64-mingw32) http-cookie (1.0.3) domain_name (~> 0.5) mime-types (3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) mysql2 (0.4.6) + mysql2 (0.4.6-x64-mingw32) netrc (0.11.0) + pg (1.2.3) + pg (1.2.3-x64-mingw32) rest-client (2.0.2) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + rest-client (2.0.2-x64-mingw32) + ffi (~> 1.9) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) sequel (4.26.0) sqlite3 (1.3.10) + sqlite3 (1.3.10-x64-mingw32) unf (0.1.4) unf_ext unf_ext (0.0.7.4) + unf_ext (0.0.7.4-x64-mingw32) PLATFORMS ruby + x64-mingw32 DEPENDENCIES mysql2 + pg rest-client sequel sqlite3 diff --git a/trac-hub b/trac-hub index d6fd5ed..d6c2570 100755 --- a/trac-hub +++ b/trac-hub @@ -1,5 +1,6 @@ #!/usr/bin/env ruby +require 'pg' require 'json' require 'logger' require 'rest-client' @@ -82,7 +83,7 @@ class Migrator def trac_mail(author) return @trac_mails_cache[author] if @trac_mails_cache.has_key?(author) # tries to retrieve the email from trac db - data = @trac.sessions.select(:value).where('name = "email" AND sid = ?', author) + data = @trac.sessions.select(:value).where(name: "email", sid: author) return (@trac_mails_cache[author] = data.first[:value]) if data.count == 1 return (@trac_mails_cache[author] = author) # not found end @@ -185,7 +186,7 @@ class Migrator # combine the changes and attachment table results and sort them by date changes = @trac.changes.where(:ticket => ticket[:id]).collect.to_a - changes += @trac.attachments.where(:type => 'ticket', :id => ticket[:id]).collect.to_a + changes += @trac.attachments.where(:type => 'ticket', :id => ticket[:id].to_s).collect.to_a changes = changes.sort_by{|x| x[:time]} # replay all changes in chronological order: From 7f983364dc1ac738aa76dc8bacfcdb714dccf615 Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Mon, 20 Dec 2021 13:22:51 +0300 Subject: [PATCH 02/19] Make the body of the issue not a comment --- trac-hub | 3 --- 1 file changed, 3 deletions(-) diff --git a/trac-hub b/trac-hub index d6c2570..8eb4e84 100755 --- a/trac-hub +++ b/trac-hub @@ -361,9 +361,6 @@ class Migrator str.gsub!(/\[changeset:(\d+)\]/) { map_changeset(Regexp.last_match[1]) } # Ticket str.gsub!(/ticket:(\d+)/, '#\1') - # set the body as a comment - str.gsub!("\n", "\n> ") - str = "> #{str}" return str end end From 183f0845ede38929227f3ea453053e7c19c0b700 Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Fri, 31 Dec 2021 10:33:19 +0300 Subject: [PATCH 03/19] Restore url of hashed attachments --- trac-hub | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trac-hub b/trac-hub index 8eb4e84..e178598 100755 --- a/trac-hub +++ b/trac-hub @@ -10,6 +10,7 @@ require 'yaml' require 'set' require 'singleton' require 'uri' +require 'digest/sha1' class GracefulQuit include Singleton @@ -300,7 +301,11 @@ class Migrator name = meta[:filename] body = meta[:description] if @attachurl - url = URI.escape("#{@attachurl}/#{meta[:id]}/#{name}") + hash_dir = Digest::SHA1.hexdigest("#{meta[:id]}") + hash_par = hash_dir[0,3] + hash_file = Digest::SHA1.hexdigest("#{name}") + file_suff = File.extname(name) + url="#{@attachurl}/#{hash_par}/#{hash_dir}/#{hash_file}#{file_suff}" text += "[`#{name}`](#{url})" if [".png", ".jpg", ".gif"].include? File.extname(name).downcase body += "\n![#{name}](#{url})" From 322ddf85584f8b759201dd94f1c881c13d5ebfa4 Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Fri, 31 Dec 2021 10:35:21 +0300 Subject: [PATCH 04/19] Print more error's info --- trac-hub | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/trac-hub b/trac-hub index e178598..b98984d 100755 --- a/trac-hub +++ b/trac-hub @@ -50,16 +50,21 @@ class Migrator @trac_mails_cache = Hash.new $logger.debug("Get highest in #{@repo}") - issues = JSON.parse(RestClient.get( - "https://api.github.com/repos/#{@repo}/issues", - {"Authorization" => "token #{@token}", - params: { - filter: "all", - state: "all", - sort: "created", - direction: "desc", - }} - )) + begin + issues = JSON.parse(RestClient.get( + "https://api.github.com/repos/#{@repo}/issues", + {"Authorization" => "token #{@token}", + params: { + filter: "all", + state: "all", + sort: "created", + direction: "desc", + }} + )) + rescue RestClient::ExceptionWithResponse => e + $logger.info("#{e.response}") + raise + end @last_created_issue = issues.empty? ? 0 : issues[0]["number"].to_i $logger.info("Last created issue on GitHub is '#{@last_created_issue}'") @@ -133,6 +138,9 @@ class Migrator {"Authorization" => "token #{@token}", "Content-Type" => "application/json", "Accept" => "application/vnd.github.golden-comet-preview+json"})) + rescue => e + $logger.info("#{e.response}") + raise end if @safetychecks @@ -143,6 +151,9 @@ class Migrator "Authorization" => "token #{@token}", "Accept" => "application/vnd.github.golden-comet-preview+json"})) end + if response["status"] == "failed" + $logger.info("#{response}") + end $logger.info("Status: #{response['status']}") issue_id = response["issue_url"].match(/\d+$/).to_s.to_i From 49f079c95e0bfc67b719c5dbbe1f7a2ac8ebdab6 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 00:02:49 +0100 Subject: [PATCH 05/19] Improve translation of Trac users to GitHub Existing code was confused about whether it wanted to use the usernames or emails and used different keys when looking up the assignee and the comment author, fix it to always use Trac usernames which seems more logical and avoids an unnecessary database lookup. Also obfuscate the emails if we do use it and also include Trac username in the string used on GitHub to allow searching by it later. --- trac-hub | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/trac-hub b/trac-hub index b98984d..5ba88c8 100755 --- a/trac-hub +++ b/trac-hub @@ -47,7 +47,7 @@ class Migrator @users = Hash[users] @labels = Hash[labels.map { |cat, rx| [cat, Hash[rx] ] }] @ticket_to_issue = {} - @trac_mails_cache = Hash.new + @trac_user_to_github_cache = Hash.new $logger.debug("Get highest in #{@repo}") begin @@ -85,13 +85,29 @@ class Migrator private - # returns the author mail if found, otherwise author itself - def trac_mail(author) - return @trac_mails_cache[author] if @trac_mails_cache.has_key?(author) - # tries to retrieve the email from trac db - data = @trac.sessions.select(:value).where(name: "email", sid: author) - return (@trac_mails_cache[author] = data.first[:value]) if data.count == 1 - return (@trac_mails_cache[author] = author) # not found + # returns the GitHub user name prefixed with "@" if a mapping for this user + # is found in "users" or the string containing user name and obfuscated + # email otherwise + def trac_user_to_github(user) + return @trac_user_to_github_cache[user] if @trac_user_to_github_cache.has_key?(user) + + # check for known users first + if @users.has_key?(user) + github_user = '@' + @users[user] + else + # tries to retrieve the email from trac db + data = @trac.sessions.select(:value).where(name: "email", sid: user) + if data.count == 1 + email = data.first[:value] + email.sub!('@', '-at-') + email.sub!('.', '-dot-') + github_user = "#{user} (#{email})" + else + github_user = user + end + end + + return (@trac_user_to_github_cache[user] = github_user) # not found end # returns the git commit hash for a specified revision (using revmap hash) @@ -234,12 +250,15 @@ class Migrator "closed" => ticket[:status] == "closed", "created_at" => format_time(ticket[:time]), } - if @users.has_key?(ticket[:owner]) - owner = trac_mail(ticket[:owner]) - github_owner = @users[owner] + owner = ticket[:owner] + if owner != nil $logger.debug("..owner in trac: #{owner}") - $logger.debug("..assignee in GitHub: #{github_owner}") - issue["assignee"] = github_owner + owner = trac_user_to_github(owner) + if owner.start_with?("@") + github_owner = owner.delete_prefix("@") + $logger.debug("..assignee in GitHub: #{github_owner}") + issue["assignee"] = github_owner + end end if ticket[:changetime] @@ -270,11 +289,7 @@ class Migrator time = Time.at(time/1e6, time%1e6) # author - author = meta[:author] - author = trac_mail(author) - if @users.has_key?(author) - author = "@" + @users[author] - end + author = trac_user_to_github(meta[:author]) text = "" text += "\n___\n" if append From a150eb801d4f5f465289c6adad99e56d47b94392 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 01:12:12 +0100 Subject: [PATCH 06/19] Add --dry-run command line option This can be useful to check that we can do everything but actually creating the tickets correctly. --- trac-hub | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/trac-hub b/trac-hub index 5ba88c8..fc0abb5 100755 --- a/trac-hub +++ b/trac-hub @@ -39,7 +39,7 @@ class GracefulQuit end class Migrator - def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks) + def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks, dryrun) @trac = trac @repo = github['repo'] @token = github["token"] @@ -73,6 +73,11 @@ class Migrator @attachurl = attachurl @singlepost = singlepost @safetychecks = safetychecks + @dryrun = dryrun + + if @dryrun + @safetychecks = false + end end def migrate(start_ticket = -1, filterout_closed = false) @@ -148,6 +153,11 @@ class Migrator # API details: https://gist.github.com/jonmagic/5282384165e0f86ef105 request = compose_issue(ticket) + if @dryrun + $logger.debug("would have posted #{request.to_json}") + next + end + response = JSON.parse(RestClient.post( "https://api.github.com/repos/#{@repo}/import/issues", request.to_json, @@ -439,6 +449,10 @@ class Options < Hash 'Import without safety-checking issue numbers.') do |fast| self[:fast] = fast end + opts.on('-n', '--dry-run', + 'Skip really creating tickets, just pretend doing it.') do |dryrun| + self[:dryrun] = dryrun + end opts.on('-o', '--opened-only', 'Skips the import of closed tickets') do |o| self[:openedonly] = o end @@ -501,6 +515,6 @@ if __FILE__ == $0 trac = Trac.new(db) migrator = Migrator.new( trac, cfg['github'], cfg['users'], cfg['labels'], revmap, - opts[:attachurl], opts[:singlepost], (not opts[:fast])) + opts[:attachurl], opts[:singlepost], (not opts[:fast]), opts[:dryrun]) migrator.migrate(opts[:start], opts[:openedonly]) end From 6b0dba29b4625cdc9782b4fc9a2045f8cd755309 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 01:21:06 +0100 Subject: [PATCH 07/19] Ignore all changes we don't handle Avoid creating useless comments without any contents for the changes we don't handle (such as those to the milestone field). --- trac-hub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trac-hub b/trac-hub index fc0abb5..f7bf8d5 100755 --- a/trac-hub +++ b/trac-hub @@ -357,7 +357,7 @@ class Migrator # so there is no need to update) text += "edited the issue description" - when 'keywords', 'cc', 'reporter', 'version' + else # don't care return nil end From d897aec71e9c2559839aea12b217ffb196acfeb1 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 01:44:37 +0100 Subject: [PATCH 08/19] Ignore changes to the "owner" field We only use the final value for the assignee and the intermediate changes are not really interesting, but leak (unobfuscated) email addresses. --- trac-hub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trac-hub b/trac-hub index f7bf8d5..9b9e4f1 100755 --- a/trac-hub +++ b/trac-hub @@ -306,7 +306,7 @@ class Migrator text += "#### #{time.strftime("%Y-%m-%d %H:%M:%S")}: #{author} " case kind - when 'owner', 'status', 'title', 'resolution', 'priority', 'component', 'type', 'severity', 'platform' + when 'status', 'title', 'resolution', 'priority', 'component', 'type', 'severity', 'platform' old = meta[:oldvalue] new = meta[:newvalue] if old and new From 0285fba697debf5e7af2391d3f8e9dae56b24161 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 01:55:39 +0100 Subject: [PATCH 09/19] Recognize Git changesets in Trac markup too Just map them to SHA-1s directly, GitHub will recognize them. --- trac-hub | 1 + 1 file changed, 1 insertion(+) diff --git a/trac-hub b/trac-hub index 9b9e4f1..6749ca8 100755 --- a/trac-hub +++ b/trac-hub @@ -400,6 +400,7 @@ class Migrator str.gsub!(/\br(\d+)\b/) { map_changeset(Regexp.last_match[1]) } str.gsub!(/\[changeset:"(\d+)".*\]/) { map_changeset(Regexp.last_match[1]) } str.gsub!(/\[changeset:(\d+)\]/) { map_changeset(Regexp.last_match[1]) } + str.gsub!(/\[changeset:"([0-9a-f]{40})\/[^\]]+\]/, '\1') # Ticket str.gsub!(/ticket:(\d+)/, '#\1') return str From 8368eca2887e004addbd487f4c43cc1369157f86 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 01:56:37 +0100 Subject: [PATCH 10/19] Recombine parts of the same change in a single comment Trac stores changes to each field separately, but presents them as a single change at the UI level, so do the same thing for the issues that we create. --- trac-hub | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/trac-hub b/trac-hub index 6749ca8..014259e 100755 --- a/trac-hub +++ b/trac-hub @@ -228,10 +228,32 @@ class Migrator changes = changes.sort_by{|x| x[:time]} # replay all changes in chronological order: - comments = changes.map{|x| ticket_change(@singlepost, x)}.select{|x| x}.to_a - if @singlepost - body += comments.map{|x| x["body"]}.join("\n") - comments = [] + comments = [] + last_change_time = nil + changes.each do |x| + # combine changes from a single change into a single comment if we're + # combining everything anyhow or if this change was done at the same + # moment as the previous one + if @singlepost + body += "\n___\n\n" + append = true + else + append = x[:time] == last_change_time + end + + comment = ticket_change(append, x) + next if not comment + + if @singlepost + body += comment["body"] + else + if append and not comments.empty? + comments[-1]["body"] += "\n" + comment["body"] + else + comments << comment + end + last_change_time = x[:time] + end end labels = Set[] @@ -301,9 +323,7 @@ class Migrator # author author = trac_user_to_github(meta[:author]) - text = "" - text += "\n___\n" if append - text += "#### #{time.strftime("%Y-%m-%d %H:%M:%S")}: #{author} " + text = "#### #{time.strftime("%Y-%m-%d %H:%M:%S")}: #{author} " case kind when 'status', 'title', 'resolution', 'priority', 'component', 'type', 'severity', 'platform' From b15780105b0f55c3aa1053bb3a2e38130a18a11e Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 3 Jan 2022 02:27:28 +0100 Subject: [PATCH 11/19] Stop showing Trac users emails They're not shown in Trac (except to admins) and so can be reasonably considered to be private, so don't show them publicly. --- trac-hub | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/trac-hub b/trac-hub index 014259e..4b4dee4 100755 --- a/trac-hub +++ b/trac-hub @@ -100,13 +100,10 @@ class Migrator if @users.has_key?(user) github_user = '@' + @users[user] else - # tries to retrieve the email from trac db - data = @trac.sessions.select(:value).where(name: "email", sid: user) + # try to retrieve the name from trac db + data = @trac.sessions.select(:value).where(name: "name", sid: user) if data.count == 1 - email = data.first[:value] - email.sub!('@', '-at-') - email.sub!('.', '-dot-') - github_user = "#{user} (#{email})" + github_user = "#{user} (#{data.first[:value]})" else github_user = user end From 26ba89a9fdbbcb48728554111caff384c73958d7 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Wed, 5 Jan 2022 16:14:41 +0100 Subject: [PATCH 12/19] Don't use Trac user name if it is empty This is possible, i.e. there can be a row in session_attribute table with the correct sid and empty value. --- trac-hub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trac-hub b/trac-hub index 4b4dee4..d6d53e3 100755 --- a/trac-hub +++ b/trac-hub @@ -102,7 +102,7 @@ class Migrator else # try to retrieve the name from trac db data = @trac.sessions.select(:value).where(name: "name", sid: user) - if data.count == 1 + if data.count == 1 and not data.first[:value].empty? github_user = "#{user} (#{data.first[:value]})" else github_user = user From 56474e4180c13a47257af4bfece09a2e7130fb5c Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sat, 8 Jan 2022 00:42:28 +0100 Subject: [PATCH 13/19] Add --skip-assignee option This allows to run the script for testing on the repositories without the user accounts used in user mappings. --- trac-hub | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/trac-hub b/trac-hub index d6d53e3..1bdc187 100755 --- a/trac-hub +++ b/trac-hub @@ -39,7 +39,7 @@ class GracefulQuit end class Migrator - def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks, dryrun) + def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks, dryrun, skipassignee) @trac = trac @repo = github['repo'] @token = github["token"] @@ -74,6 +74,7 @@ class Migrator @singlepost = singlepost @safetychecks = safetychecks @dryrun = dryrun + @skipassignee = skipassignee if @dryrun @safetychecks = false @@ -279,14 +280,17 @@ class Migrator "closed" => ticket[:status] == "closed", "created_at" => format_time(ticket[:time]), } - owner = ticket[:owner] - if owner != nil - $logger.debug("..owner in trac: #{owner}") - owner = trac_user_to_github(owner) - if owner.start_with?("@") - github_owner = owner.delete_prefix("@") - $logger.debug("..assignee in GitHub: #{github_owner}") - issue["assignee"] = github_owner + + if not @skipassignee + owner = ticket[:owner] + if owner != nil + $logger.debug("..owner in trac: #{owner}") + owner = trac_user_to_github(owner) + if owner.start_with?("@") + github_owner = owner.delete_prefix("@") + $logger.debug("..assignee in GitHub: #{github_owner}") + issue["assignee"] = github_owner + end end end @@ -455,6 +459,10 @@ class Options < Hash 'allows to specify a commit revision mapping FILE') do |file| self[:revmapfile] = file end + opts.on('-A', '--skip-assignee', + 'Skip assigning tickets, even if user mapping is found') do |skipassignee| + self[:skipassignee] = skipassignee + end opts.on('-a', '--attachment-url URL', 'if attachment files are reachable via a URL we reference this here') do |url| self[:attachurl] = url @@ -533,6 +541,7 @@ if __FILE__ == $0 trac = Trac.new(db) migrator = Migrator.new( trac, cfg['github'], cfg['users'], cfg['labels'], revmap, - opts[:attachurl], opts[:singlepost], (not opts[:fast]), opts[:dryrun]) + opts[:attachurl], opts[:singlepost], (not opts[:fast]), + opts[:dryrun], opts[:skipassignee]) migrator.migrate(opts[:start], opts[:openedonly]) end From 99828d013da72852e9daee0ad5da06f477b23efd Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sat, 8 Jan 2022 00:47:20 +0100 Subject: [PATCH 14/19] Allow specify milestones mapping and use it if defined Importing milestones may be useful too. --- trac-hub | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/trac-hub b/trac-hub index 1bdc187..d49fcbb 100755 --- a/trac-hub +++ b/trac-hub @@ -39,13 +39,14 @@ class GracefulQuit end class Migrator - def initialize(trac, github, users, labels, revmap, attachurl, singlepost, safetychecks, dryrun, skipassignee) + def initialize(trac, github, users, labels, milestones, revmap, attachurl, singlepost, safetychecks, dryrun, skipassignee) @trac = trac @repo = github['repo'] @token = github["token"] @users = Hash[users] @labels = Hash[labels.map { |cat, rx| [cat, Hash[rx] ] }] + @milestones = milestones @ticket_to_issue = {} @trac_user_to_github_cache = Hash.new @@ -294,6 +295,10 @@ class Migrator end end + if @milestones.has_key?(ticket[:milestone]) + issue["milestone"] = @milestones[ticket[:milestone]] + end + if ticket[:changetime] issue["updated_at"] = format_time(ticket[:changetime]) end @@ -540,7 +545,8 @@ if __FILE__ == $0 trac = Trac.new(db) migrator = Migrator.new( - trac, cfg['github'], cfg['users'], cfg['labels'], revmap, + trac, cfg['github'], cfg['users'], cfg['labels'], cfg['milestones'], + revmap, opts[:attachurl], opts[:singlepost], (not opts[:fast]), opts[:dryrun], opts[:skipassignee]) migrator.migrate(opts[:start], opts[:openedonly]) From 24bf980cbb7242a98d4f02950499f07b61c7ffbb Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Sat, 8 Jan 2022 15:54:09 +0100 Subject: [PATCH 15/19] Add --stop-at option This is useful to test migration of a single ticket, for example. --- trac-hub | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/trac-hub b/trac-hub index d49fcbb..60a4c71 100755 --- a/trac-hub +++ b/trac-hub @@ -82,12 +82,12 @@ class Migrator end end - def migrate(start_ticket = -1, filterout_closed = false) + def migrate(start_ticket = -1, end_ticket = -1, filterout_closed = false) if start_ticket == -1 start_ticket = @last_created_issue+1 end GracefulQuit.enable - migrate_tickets(start_ticket, filterout_closed) + migrate_tickets(start_ticket, end_ticket.to_i, filterout_closed) end private @@ -130,10 +130,11 @@ class Migrator end # Creates github issues for trac tickets. - def migrate_tickets(start_ticket, filterout_closed) + def migrate_tickets(start_ticket, end_ticket, filterout_closed) $logger.info('migrating issues') # We match the issue title to determine whether an issue exists already. @trac.tickets.order(:id).where{id >= start_ticket}.all.each do |ticket| + break if end_ticket != -1 and ticket[:id] > end_ticket next if filterout_closed and ticket[:status] == "closed" GracefulQuit.check("quitting after processing ticket ##{@last_created_issue}") @@ -460,6 +461,9 @@ class Options < Hash opts.on('-s', '--start-at ID', 'start migration from ticket with number ') do |id| self[:start] = id end + opts.on('-e', '--stop-at ID', 'stop migration at ticket with number ') do |id| + self[:end] = id + end opts.on('-r', '--rev-map-file FILE', 'allows to specify a commit revision mapping FILE') do |file| self[:revmapfile] = file @@ -498,6 +502,7 @@ class Options < Hash self[:config] = default end self[:start] = -1 unless self[:start] + self[:end] = -1 unless self[:end] rescue => e STDERR.puts(e) STDERR.puts('run with -h to see available options') @@ -549,5 +554,5 @@ if __FILE__ == $0 revmap, opts[:attachurl], opts[:singlepost], (not opts[:fast]), opts[:dryrun], opts[:skipassignee]) - migrator.migrate(opts[:start], opts[:openedonly]) + migrator.migrate(opts[:start], opts[:end], opts[:openedonly]) end From ef624ee797fc16134b9c9cd7809b3feea31d911d Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Wed, 5 Jan 2022 16:15:15 +0100 Subject: [PATCH 16/19] Sleep longer before checking for import status Checking too often is counterproductive and just results in exhausting the request quota. --- trac-hub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trac-hub b/trac-hub index 60a4c71..dfd3c52 100755 --- a/trac-hub +++ b/trac-hub @@ -171,7 +171,7 @@ class Migrator if @safetychecks while response["status"] == "pending" - sleep 0.1 + sleep 1.8 $logger.info("Checking import status: #{response['id']}") response = JSON.parse(RestClient.get(response['url'], { "Authorization" => "token #{@token}", From a38107e8c9250b4adefae5178ef9f12ea2e9bdb9 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 10 Jan 2022 00:34:10 +0100 Subject: [PATCH 17/19] Add possibility to map some Trac keywords to GitHub labels Doing it for all keywords is not a good idea as there are typically too many of them, but it can be still worth doing it for some selected keywords, that can now be specified in the labels config file section. --- trac-hub | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/trac-hub b/trac-hub index dfd3c52..d057849 100755 --- a/trac-hub +++ b/trac-hub @@ -272,6 +272,14 @@ class Migrator labels.add(@labels.fetch('priority', Hash[])[ticket[:priority]]) labels.add(@labels.fetch('severity', Hash[])[ticket[:severity]]) labels.add(@labels.fetch('version', Hash[])[ticket[:version]]) + + if ticket[:keywords] + keywords = @labels.fetch('keywords', Hash[]) + ticket[:keywords].split(/[,;]? +/).each do |kw| + labels.add(keywords[kw]) + end + end + # If the field is not set, it will be nil and generate an unprocessable json labels.delete(nil) From e7332bf3218e7fb4b5272ab2e01917a2006a82e4 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Mon, 10 Jan 2022 00:43:55 +0100 Subject: [PATCH 18/19] Allow mapping some Trac status values to GitHub labels too This can be useful when using custom statuses, e.g. "confirmed" for the tickets that were verified to be reproducible. --- trac-hub | 1 + 1 file changed, 1 insertion(+) diff --git a/trac-hub b/trac-hub index d057849..978fbb1 100755 --- a/trac-hub +++ b/trac-hub @@ -267,6 +267,7 @@ class Migrator end end labels.add(@labels.fetch('component', Hash[])[ticket[:component]]) + labels.add(@labels.fetch('status', Hash[])[ticket[:status]]) labels.add(@labels.fetch('type', Hash[])[ticket[:type]]) labels.add(@labels.fetch('resolution', Hash[])[ticket[:resolution]]) labels.add(@labels.fetch('priority', Hash[])[ticket[:priority]]) From 6120b0986fb85ef219d4a4958e5d8c9425dad025 Mon Sep 17 00:00:00 2001 From: Vadim Zeitlin Date: Wed, 12 Jan 2022 23:48:40 +0100 Subject: [PATCH 19/19] Allow using an offset for the issue numbers This is helpful when importing issues into an existing repository with some existing PRs, as they take consume numbers in the same namespace as the issues, making it impossible to preserve (all) the existing issue numbers. Using this option it's at least possible to start importing from some round number, e.g. 100, or 1000, so that the old ticket N is now mapped to 100+N or 1000+N. --- trac-hub | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/trac-hub b/trac-hub index 978fbb1..3706642 100755 --- a/trac-hub +++ b/trac-hub @@ -39,7 +39,7 @@ class GracefulQuit end class Migrator - def initialize(trac, github, users, labels, milestones, revmap, attachurl, singlepost, safetychecks, dryrun, skipassignee) + def initialize(trac, github, users, labels, milestones, revmap, attachurl, singlepost, safetychecks, dryrun, skipassignee, ticketoffset) @trac = trac @repo = github['repo'] @token = github["token"] @@ -76,6 +76,7 @@ class Migrator @safetychecks = safetychecks @dryrun = dryrun @skipassignee = skipassignee + @ticketoffset = ticketoffset if @dryrun @safetychecks = false @@ -138,7 +139,7 @@ class Migrator next if filterout_closed and ticket[:status] == "closed" GracefulQuit.check("quitting after processing ticket ##{@last_created_issue}") - if @safetychecks; begin + if @safetychecks and not @ticketoffset; begin # issue exists already: issue = JSON.parse(RestClient.get( "https://api.github.com/repos/#{@repo}/issues/#{ticket[:id]}", @@ -186,7 +187,7 @@ class Migrator $logger.info("created issue ##{issue_id} for ticket #{ticket[:id]}") # assert correct issue number - if issue_id != ticket[:id] + if issue_id != ticket[:id] + @ticketoffset $logger.info("mismatch issue ##{issue_id} for ticket #{ticket[:id]}") exit 1 end @@ -473,6 +474,10 @@ class Options < Hash opts.on('-e', '--stop-at ID', 'stop migration at ticket with number ') do |id| self[:end] = id end + opts.on('-O', '--ticket-offset OFFSET', + 'Use the given offset when checking ticket number') do |ticketoffset| + self[:ticketoffset] = ticketoffset.to_i + end opts.on('-r', '--rev-map-file FILE', 'allows to specify a commit revision mapping FILE') do |file| self[:revmapfile] = file @@ -562,6 +567,6 @@ if __FILE__ == $0 trac, cfg['github'], cfg['users'], cfg['labels'], cfg['milestones'], revmap, opts[:attachurl], opts[:singlepost], (not opts[:fast]), - opts[:dryrun], opts[:skipassignee]) + opts[:dryrun], opts[:skipassignee], opts[:ticketoffset]) migrator.migrate(opts[:start], opts[:end], opts[:openedonly]) end