Merge git repositories – Code musings in C#

We have more than 20 Git repositories for the same project, but it causes lots of headaches for building. We’ve decided to merge most of them into a single one, keeping the file history when possible.

This script uses newren/git-filter-repo and as such needs Python 3.

It’s done in 3 steps:

clone the local repositories from d:\dev\xxx to d:\devnew\xxx using git-filter-repo with source and target parameters
create a new repository at d:\devnew\merged
merge from d:\devnew\xxx to d:\devnew\merged\xxx using git merge --allow-unrelated-histories

$repos = @(
    "Build",
    "Deploy",
    "External",
    "API",
    "Database",
    "Server",
    "Services",
    "UI",
    "Utils"
)

$currentLocation = "D:\Dev"
$newLocation = "D:\DevNew"

# removed:
# - check that git and python3 are installed and on the PATH
# - cleanup previous merges
# - check that repositories are clean
# - logging/write-host

$total = $repos.Length
$nb = 0

foreach ($repo in $repos) {
	$nb++
	Write-Host "Copy of $repo ($nb/$total)" -ForegroundColor Blue

	# checkout all remote branches in local
	git -C $currentLocation\$repo fetch -p
	$remoteBranches = (git -C $currentLocation\$repo branch -l -r).Split("`n").Trim().Replace("origin/","") | where { -not $_.StartsWith("HEAD ->") }
	$localBranches = (git -C $currentLocation\$repo branch -l).Split("`n").Replace("*","").Trim()
	foreach ($remoteBranch in $remoteBranches) {
		if (-not ($remoteBranch -in $localBranches)) {	
			git -C $currentLocation\$repo branch $remoteBranch "origin/$remoteBranch"
		}
		else {
			git -C $currentLocation\$repo rebase "origin/$remoteBranch" $remoteBranch
		}
	}

	# re-checkout dev
	git -C $currentLocation\$repo checkout dev
	git -C $currentLocation\$repo fetch
	git -C $currentLocation\$repo rebase
	
	# https://stackoverflow.com/a/14728706/
	git -C $currentLocation\$repo -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 -c gc.rerereunresolved=0 -c gc.pruneExpire=now gc

	# initialize new repo
	git init $newLocation\$repo
	
	# define ignored files and folders in history (old data/tests/mistakes you won't ever need again)
	$ignores = @()
	elseif ($repo -eq "Build") {
		$ignores = @("installers", "xunit", "opencover", "reportgenerator")
	}
	elseif ($repo -eq "Server") {
		$ignores = @("TestProject", "OldProject")
	}
	
	# build filter string
	$filterExpression = "python git-filter-repo --source $currentLocation\$repo --target $newLocation\$repo --path-rename :$repo/ --path packages --path trunk --path-glob *.suo"
	foreach ($ignore in $ignores) {
		if ($ignore.IndexOf("*") -gt -1) {
			$filterExpression += " --path-glob $ignore"
		}
		else {
			$filterExpression += " --path $ignore"
		}
	}
	
	# move tags/* and branches/* to $repo/tags/* and $repo/branches/*
	$filterExpression += " --invert-paths --mailmap $PSScriptRoot\mailmap.txt --refname-callback 'return refname.replace(`"refs/heads/`", `"refs/heads/$repo/`")replace(`"refs/tags/`", `"refs/tags/$repo/`")'"
	
	# copy the git history while filtering the repo
	Invoke-Expression $filterExpression
}

# create merged repo
git init $newLocation\Merged
git -C $newLocation\Merged checkout -b dev

# initialize LFS for binary files
git -C $newLocation\Merged lfs install
git -C $newLocation\Merged lfs track "*.dll"
git -C $newLocation\Merged lfs track "*.exe"

# operations on new repositories
$nb = 0
foreach ($repo in $repos) {
	$nb++
	Write-Host "Starts merging $repo into $newLocation\Merged ($nb/$total)" -ForegroundColor Blue

	# get list of imported branches and tags
	$branches = (git -C $newLocation\$repo branch -l).Split("`n").Replace("*","").Trim() | where { $_.StartsWith($repo) }
	$tags = $(git -C $newLocation\$repo tag -l).Split()

	# add remote
	git -C $newLocation\Merged remote add "$($repo)_remote" $newLocation\$repo
	git -C $newLocation\Merged fetch "$($repo)_remote"
	
	# create branches
	foreach ($branch in $branches) {
		git -C $newLocation\Merged branch $branch "$($repo)_remote/$branch"
	}
	
	# merge imported repo into merged repo
	git -C $newLocation\Merged checkout dev
	git -C $newLocation\Merged merge --allow-unrelated-histories "$repo/dev"

	git -C $newLocation\Merged remote remove "$($repo)_remote"
}

git -C $newLocation\Merged -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 -c gc.rerereunresolved=0 -c gc.pruneExpire=now gc

$EndDate = Get-Date

$duration = New-Timespan -Start $StartDate -End $EndDate
Write-Host "====================== DONE IN $($duration.minutes) mn ======================" -ForegroundColor Green

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

$repos = @(

"Build",

"Deploy",

"External",

"API",

"Database",

"Server",

"Services",

"UI",

"Utils"

)

$currentLocation = "D:\Dev"

$newLocation = "D:\DevNew"

# removed:

# - check that git and python3 are installed and on the PATH

# - cleanup previous merges

# - check that repositories are clean

# - logging/write-host

$total = $repos.Length

$nb = 0

foreach ($repo in $repos) {

$nb++

Write-Host "Copy of $repo ($nb/$total)" -ForegroundColor Blue

# checkout all remote branches in local

git -C $currentLocation\$repo fetch -p

$remoteBranches = (git -C $currentLocation\$repo branch -l -r).Split("`n").Trim().Replace("origin/","") | where { -not $_.StartsWith("HEAD ->") }

$localBranches = (git -C $currentLocation\$repo branch -l).Split("`n").Replace("*","").Trim()

foreach ($remoteBranch in $remoteBranches) {

if (-not ($remoteBranch -in $localBranches)) {

git -C $currentLocation\$repo branch $remoteBranch "origin/$remoteBranch"

}

else {

git -C $currentLocation\$repo rebase "origin/$remoteBranch" $remoteBranch

}

# re-checkout dev

git -C $currentLocation\$repo checkout dev

git -C $currentLocation\$repo fetch

git -C $currentLocation\$repo rebase

# https://stackoverflow.com/a/14728706/

git -C $currentLocation\$repo -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 -c gc.rerereunresolved=0 -c gc.pruneExpire=now gc

# initialize new repo

git init $newLocation\$repo

# define ignored files and folders in history (old data/tests/mistakes you won't ever need again)

$ignores = @()

elseif ($repo -eq "Build") {

$ignores = @("installers", "xunit", "opencover", "reportgenerator")

}

elseif ($repo -eq "Server") {

$ignores = @("TestProject", "OldProject")

}

# build filter string

$filterExpression = "python git-filter-repo --source $currentLocation\$repo --target $newLocation\$repo --path-rename :$repo/ --path packages --path trunk --path-glob *.suo"

foreach ($ignore in $ignores) {

if ($ignore.IndexOf("*") -gt -1) {

$filterExpression += " --path-glob $ignore"

}

else {

$filterExpression += " --path $ignore"

}

# move tags/* and branches/* to $repo/tags/* and $repo/branches/*

$filterExpression += " --invert-paths --mailmap $PSScriptRoot\mailmap.txt --refname-callback 'return refname.replace(`"refs/heads/`", `"refs/heads/$repo/`")replace(`"refs/tags/`", `"refs/tags/$repo/`")'"

# copy the git history while filtering the repo

Invoke-Expression $filterExpression

}

# create merged repo

git init $newLocation\Merged

git -C $newLocation\Merged checkout -b dev

# initialize LFS for binary files

git -C $newLocation\Merged lfs install

git -C $newLocation\Merged lfs track "*.dll"

git -C $newLocation\Merged lfs track "*.exe"

# operations on new repositories

$nb = 0

foreach ($repo in $repos) {

$nb++

Write-Host "Starts merging $repo into $newLocation\Merged ($nb/$total)" -ForegroundColor Blue

# get list of imported branches and tags

$branches = (git -C $newLocation\$repo branch -l).Split("`n").Replace("*","").Trim() | where { $_.StartsWith($repo) }

$tags = $(git -C $newLocation\$repo tag -l).Split()

# add remote

git -C $newLocation\Merged remote add "$($repo)_remote" $newLocation\$repo

git -C $newLocation\Merged fetch "$($repo)_remote"

# create branches

foreach ($branch in $branches) {

git -C $newLocation\Merged branch $branch "$($repo)_remote/$branch"

}

# merge imported repo into merged repo

git -C $newLocation\Merged checkout dev

git -C $newLocation\Merged merge --allow-unrelated-histories "$repo/dev"

git -C $newLocation\Merged remote remove "$($repo)_remote"

}

git -C $newLocation\Merged -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 -c gc.rerereunresolved=0 -c gc.pruneExpire=now gc

$EndDate = Get-Date

$duration = New-Timespan -Start $StartDate -End $EndDate

Write-Host "====================== DONE IN $($duration.minutes) mn ======================" -ForegroundColor Green

I’ve removed a lot of code for brievity, so maybe it won’t work out of the box, but you should get the general idea.

Next up: pushing the new repository, then migrate the developers workstations, for which I wrote another script:

# removed: checks and logs

$oldRootPath = "D:\Dev\"
$newRootPath = "D:\Dev\Mezzo\"

Install-Module -Name 'IISAdministration' -Force -ErrorAction Stop
Import-Module IISAdministration

# get sites list
$sites = @()
Get-IISSite | % {
	$sitepath = $_.Applications.VirtualDirectories.PhysicalPath

	if ($sitepath -is [array]) {
		$sitepath = $sitepath[0]
	}

	$sites += @{
		PhysicalPath = $sitepath
		Name = $_.Name
	}
}

# iterate over sites
foreach ($site in $sites) {
	if (-not $site.PhysicalPath.ToLower().StartsWith($oldRootPath.Tolower())) {
		continue
	}
	
	$sitename = $site.Name
	$sitepath = $site.PhysicalPath

	$relativePhysicalPath = $sitepath.Substring($oldRootPath.Length)
	$skip = $false
	$virtualDirectoryNames = @()
	switch ($relativePhysicalPath.ToLower()) {
		"ui\users" {
			$virtualDirectoryNames = @("js")
		}
		"ui\admin" {
			$virtualDirectoryNames = @("js")
		}
	}
	
	$newPhysicalPath = "$newRootPath$relativePhysicalPath"

	$manager = Get-IISServerManager
	$rootsite = $manager.Sites[$sitename].Applications["/"].VirtualDirectories["/"]
	$rootsite.PhysicalPath = $newPhysicalPath
	
	foreach ($vdirname in $virtualDirectoryNames) {
		$vdirname = "/" + ($vdirname -replace "\\", "/")

		$vdir = $manager.Sites[$sitename].Applications["/"].VirtualDirectories[$vdirname]
		if ($vdir -eq $null) {
			continue
		}

		$relativePhysicalPath = $vdir.Attributes["physicalPath"].Value.Substring($oldRootPath.Length)
		$newPhysicalPath = "$newRootPath$relativePhysicalPath"
		$vdir.PhysicalPath = $newPhysicalPath
	}
	
	$manager.CommitChanges()
}

# restart IIS
& iisreset

# removed: checks and logs

$oldRootPath = "D:\Dev\"

$newRootPath = "D:\Dev\Mezzo\"

Install-Module -Name 'IISAdministration' -Force -ErrorAction Stop

Import-Module IISAdministration

# get sites list

$sites = @()

Get-IISSite | % {

$sitepath = $_.Applications.VirtualDirectories.PhysicalPath

if ($sitepath -is [array]) {

$sitepath = $sitepath[0]

}

$sites += @{

PhysicalPath = $sitepath

Name = $_.Name

}

# iterate over sites

foreach ($site in $sites) {

if (-not $site.PhysicalPath.ToLower().StartsWith($oldRootPath.Tolower())) {

continue

}

$sitename = $site.Name

$sitepath = $site.PhysicalPath

$relativePhysicalPath = $sitepath.Substring($oldRootPath.Length)

$skip = $false

$virtualDirectoryNames = @()

switch ($relativePhysicalPath.ToLower()) {

"ui\users" {

$virtualDirectoryNames = @("js")

}

"ui\admin" {

$virtualDirectoryNames = @("js")

}

$newPhysicalPath = "$newRootPath$relativePhysicalPath"

$manager = Get-IISServerManager

$rootsite = $manager.Sites[$sitename].Applications["/"].VirtualDirectories["/"]

$rootsite.PhysicalPath = $newPhysicalPath

foreach ($vdirname in $virtualDirectoryNames) {

$vdirname = "/" + ($vdirname -replace "\\", "/")

$vdir = $manager.Sites[$sitename].Applications["/"].VirtualDirectories[$vdirname]

if ($vdir -eq $null) {

continue

}

$relativePhysicalPath = $vdir.Attributes["physicalPath"].Value.Substring($oldRootPath.Length)

$newPhysicalPath = "$newRootPath$relativePhysicalPath"

$vdir.PhysicalPath = $newPhysicalPath

}

$manager.CommitChanges()

}

# restart IIS

& iisreset

Next up: migrating Jenkins jobs. With more than 170 jobs, doing it by hand is a real chore. Fortunately, you can also automate it.

# removed: variables, checks, etc

# - generate an API token in your Jenkins profile
# - create a credentials.txt file, write your login like: `login:tokenapi`
# - call: certutil -encode credentials.txt credentials.asc
# - remove the lines `BEGIN CERTIFICATE` and `END CERTIFICATE`
$credFile = Get-Content ".\credentials.asc"
if ($credFile -contains "-----BEGIN CERTIFICATE-----") {
	Write-Host "Remove the 'BEGIN CERTIFICATE' and 'END CERTIFICATE' lines from credentials.asc"
	Exit 1
}

$Credentials = $credFile.Trim()
$authHeader = @{Authorization = ('Basic ' + $Credentials) }

# list projects
$projects = Invoke-RestMethod -Method "GET" -Uri "$Url/api/json" -Headers $authHeader -ContentType "application/json; charset=utf-8"

# for each project, get the config.xml file and save it here
foreach ($job in $projects.jobs) {
    $name = $job.name
    $configUrl = "$Url/job/$name/config.xml"

    Invoke-RestMethod -Method "GET" -Uri $configUrl -Headers $authHeader -ContentType "text/xml" | Out-File "jobs\$name.xml"
}

# removed: variables, checks, etc

# - generate an API token in your Jenkins profile

# - create a credentials.txt file, write your login like: `login:tokenapi`

# - call: certutil -encode credentials.txt credentials.asc

# - remove the lines `BEGIN CERTIFICATE` and `END CERTIFICATE`

$credFile = Get-Content ".\credentials.asc"

if ($credFile -contains "-----BEGIN CERTIFICATE-----") {

Write-Host "Remove the 'BEGIN CERTIFICATE' and 'END CERTIFICATE' lines from credentials.asc"

Exit 1

}

$Credentials = $credFile.Trim()

$authHeader = @{Authorization = ('Basic ' + $Credentials) }

# list projects

$projects = Invoke-RestMethod -Method "GET" -Uri "$Url/api/json" -Headers $authHeader -ContentType "application/json; charset=utf-8"

# for each project, get the config.xml file and save it here

foreach ($job in $projects.jobs) {

$name = $job.name

$configUrl = "$Url/job/$name/config.xml"

Invoke-RestMethod -Method "GET" -Uri $configUrl -Headers $authHeader -ContentType "text/xml" | Out-File "jobs\$name.xml"

}

Now we can version the Jenkins configs. Then we edit, and update them.

# removed: variables, checks, etc

$credFile = Get-Content ".\credentials.asc"
if ($credFile -contains "-----BEGIN CERTIFICATE-----") {
	Write-Host "Remove the 'BEGIN CERTIFICATE' and 'END CERTIFICATE' lines from credentials.asc"
	Exit 1
}

$Credentials = $credFile.Trim()
$authHeader = @{Authorization = ('Basic ' + $Credentials) }

$configUrl = "$Url/job/$JobName/config.xml"

# push $localConfigFile to $Url
Invoke-RestMethod -Method "POST" -Uri $configUrl -Headers $authHeader -ContentType "application/xml" -InFile $localConfigFile