From 60da4b4a41f4d4726a4773476c49a721a8875e62 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Tue, 7 Apr 2026 14:50:25 -0700 Subject: [PATCH 01/71] Initial commit for gl2gh. Copy and adapt bbs2gh for GitLab. --- src/Octoshift/Models/GitlabRepository.cs | 8 + .../Services/EnvironmentVariableProvider.cs | 8 + src/Octoshift/Services/GithubApi.cs | 41 + src/Octoshift/Services/GitlabApi.cs | 148 +++ src/Octoshift/Services/GitlabClient.cs | 116 ++ .../OctoshiftCLI.Tests.csproj | 4 + .../AbortMigrationCommandTests.cs | 22 + .../CreateTeam/CreateTeamCommandTests.cs | 23 + .../DownloadLogs/DownloadLogsCommandTests.cs | 38 + .../GenerateMannequinCsvCommandTests.cs | 23 + .../GenerateScriptCommandArgsTests.cs | 65 + .../GenerateScriptCommandHandlerTests.cs | 839 +++++++++++++ .../GenerateScriptCommandTests.cs | 129 ++ .../GrantMigratorRoleCommandTests.cs | 24 + .../InventoryReportCommandHandlerTests.cs | 132 ++ .../InventoryReportCommandTests.cs | 51 + .../MigrateRepoCommandArgsTests.cs | 652 ++++++++++ .../MigrateRepoCommandHandlerTests.cs | 1079 +++++++++++++++++ .../MigrateRepo/MigrateRepoCommandTests.cs | 357 ++++++ .../ReclaimMannequinCommandTests.cs | 28 + .../RevokeMigratorRoleCommandTests.cs | 24 + .../WaitForMigrationCommandTests.cs | 22 + .../gl2gh/Factories/GitlabApiFactoryTests.cs | 93 ++ .../Services/GitlabInspectorServiceTests.cs | 171 +++ .../GitlabSmbArchiveDownloaderTests.cs | 191 +++ .../GitlabSshArchiveDownloaderTests.cs | 85 ++ .../ProjectsCsvGeneratorServiceTests.cs | 92 ++ .../Services/ReposCsvGeneratorServiceTests.cs | 186 +++ src/OctoshiftCLI.sln | 6 + .../AbortMigration/AbortMigrationCommand.cs | 8 + .../Commands/CreateTeam/CreateTeamCommand.cs | 8 + .../DownloadLogs/DownloadLogsCommand.cs | 17 + .../GenerateMannequinCsvCommand.cs | 8 + .../GenerateScript/GenerateScriptCommand.cs | 159 +++ .../GenerateScriptCommandArgs.cs | 55 + .../GenerateScriptCommandHandler.cs | 213 ++++ .../GrantMigratorRoleCommand.cs | 8 + .../InventoryReport/InventoryReportCommand.cs | 83 ++ .../InventoryReportCommandArgs.cs | 15 + .../InventoryReportCommandHandler.cs | 66 + .../MigrateRepo/MigrateRepoCommand.cs | 271 +++++ .../MigrateRepo/MigrateRepoCommandArgs.cs | 186 +++ .../MigrateRepo/MigrateRepoCommandHandler.cs | 403 ++++++ .../ReclaimMannequinCommand.cs | 8 + .../RevokeMigratorRoleCommand.cs | 8 + .../WaitForMigrationCommand.cs | 7 + src/gl2gh/ExportState.cs | 12 + src/gl2gh/Factories/GitlabApiFactory.cs | 44 + .../GitlabArchiveDownloaderFactory.cs | 30 + .../GitlabInspectorServiceFactory.cs | 18 + src/gl2gh/GitlabSettings.cs | 8 + src/gl2gh/Program.cs | 154 +++ src/gl2gh/Services/GitlabInspectorService.cs | 97 ++ .../Services/GitlabSmbArchiveDownloader.cs | 229 ++++ .../Services/GitlabSshArchiveDownloader.cs | 127 ++ src/gl2gh/Services/IBbsArchiveDownloader.cs | 20 + .../Services/ProjectsCsvGeneratorService.cs | 47 + .../Services/ReposCsvGeneratorService.cs | 70 ++ src/gl2gh/gl2gh.csproj | 25 + 59 files changed, 7061 insertions(+) create mode 100644 src/Octoshift/Models/GitlabRepository.cs create mode 100644 src/Octoshift/Services/GitlabApi.cs create mode 100644 src/Octoshift/Services/GitlabClient.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs create mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs create mode 100644 src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs create mode 100644 src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs create mode 100644 src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs create mode 100644 src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs create mode 100644 src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs create mode 100644 src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs create mode 100644 src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs create mode 100644 src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs create mode 100644 src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs create mode 100644 src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs create mode 100644 src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs create mode 100644 src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs create mode 100644 src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs create mode 100644 src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs create mode 100644 src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs create mode 100644 src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs create mode 100644 src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs create mode 100644 src/gl2gh/ExportState.cs create mode 100644 src/gl2gh/Factories/GitlabApiFactory.cs create mode 100644 src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs create mode 100644 src/gl2gh/Factories/GitlabInspectorServiceFactory.cs create mode 100644 src/gl2gh/GitlabSettings.cs create mode 100644 src/gl2gh/Program.cs create mode 100644 src/gl2gh/Services/GitlabInspectorService.cs create mode 100644 src/gl2gh/Services/GitlabSmbArchiveDownloader.cs create mode 100644 src/gl2gh/Services/GitlabSshArchiveDownloader.cs create mode 100644 src/gl2gh/Services/IBbsArchiveDownloader.cs create mode 100644 src/gl2gh/Services/ProjectsCsvGeneratorService.cs create mode 100644 src/gl2gh/Services/ReposCsvGeneratorService.cs create mode 100644 src/gl2gh/gl2gh.csproj diff --git a/src/Octoshift/Models/GitlabRepository.cs b/src/Octoshift/Models/GitlabRepository.cs new file mode 100644 index 000000000..c322280c9 --- /dev/null +++ b/src/Octoshift/Models/GitlabRepository.cs @@ -0,0 +1,8 @@ +namespace Octoshift.Models; + +public record GitlabRepository +{ + public string Id { get; init; } + public string Name { get; init; } + public string Slug { get; init; } +} diff --git a/src/Octoshift/Services/EnvironmentVariableProvider.cs b/src/Octoshift/Services/EnvironmentVariableProvider.cs index e59b47109..f1a3f7d70 100644 --- a/src/Octoshift/Services/EnvironmentVariableProvider.cs +++ b/src/Octoshift/Services/EnvironmentVariableProvider.cs @@ -15,6 +15,8 @@ public class EnvironmentVariableProvider private const string AWS_REGION = "AWS_REGION"; private const string BBS_USERNAME = "BBS_USERNAME"; private const string BBS_PASSWORD = "BBS_PASSWORD"; + private const string GITLAB_USERNAME = "GITLAB_USERNAME"; + private const string GITLAB_PASSWORD = "GITLAB_PASSWORD"; private const string SMB_PASSWORD = "SMB_PASSWORD"; private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK"; private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK"; @@ -57,6 +59,12 @@ public virtual string BbsUsername(bool throwIfNotFound = true) => public virtual string BbsPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); + public virtual string GitlabUsername(bool throwIfNotFound = true) => + GetSecret(BBS_USERNAME, throwIfNotFound); + + public virtual string GitlabPassword(bool throwIfNotFound = true) => + GetSecret(BBS_PASSWORD, throwIfNotFound); + public virtual string SmbPassword(bool throwIfNotFound = true) => GetSecret(SMB_PASSWORD, throwIfNotFound); diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 00bf29e4e..822fb7a43 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -346,6 +346,30 @@ public virtual async Task CreateBbsMigrationSource(string orgId) return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; } + public virtual async Task CreateGitlabMigrationSource(string orgId) + { + var url = $"{_apiUrl}/graphql"; + + var query = "mutation createMigrationSource($name: String!, $url: String!, $ownerId: ID!, $type: MigrationSourceType!)"; + var gql = "createMigrationSource(input: {name: $name, url: $url, ownerId: $ownerId, type: $type}) { migrationSource { id, name, url, type } }"; + + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + name = "GitLab Source", + url = "https://not-used", + ownerId = orgId, + type = "GITLAB_ARCHIVE" + }, + operationName = "createMigrationSource" + }; + + var data = await _client.PostGraphQLAsync(url, payload); + return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; + } + public virtual async Task CreateGhecMigrationSource(string orgId) { var url = $"{_apiUrl}/graphql"; @@ -534,6 +558,23 @@ public virtual async Task StartBbsMigration(string migrationSourceId, st ); } + public virtual async Task StartGitlabMigration(string migrationSourceId, string gitlabRepoUrl, string orgId, string repo, string targetToken, string archiveUrl, string targetRepoVisibility = null) + { + return await StartMigration( + migrationSourceId, + gitlabRepoUrl, // source repository URL + orgId, + repo, + "not-used", // source access token + targetToken, + archiveUrl, + "https://not-used", // metadata archive URL + false, // skip releases + targetRepoVisibility, + false // lock source + ); + } + public virtual async Task<(string State, string RepositoryName, int WarningsCount, string FailureReason, string MigrationLogUrl)> GetMigration(string migrationId) { var url = $"{_apiUrl}/graphql"; diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs new file mode 100644 index 000000000..82d174d57 --- /dev/null +++ b/src/Octoshift/Services/GitlabApi.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabApi +{ + private readonly GitlabClient _client; + private readonly string _gitlabBaseUrl; + private readonly OctoLogger _log; + + public GitlabApi(GitlabClient client, string gitlabServerUrl, OctoLogger log) + { + _client = client; + _gitlabBaseUrl = gitlabServerUrl?.TrimEnd('/'); + _log = log; + } + + public virtual async Task GetServerVersion() + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/application-properties"; + + var content = await _client.GetAsync(url); + + return (string)JObject.Parse(content)["version"]; + } + + public virtual async Task StartExport(string projectKey, string slug) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/migration/exports"; + var payload = new + { + repositoriesRequest = new + { + includes = new[] + { + new + { + projectKey, + slug + } + } + } + }; + + var content = await _client.PostAsync(url, payload); + + return (long)JObject.Parse(content)["id"]; + } + + public virtual async Task<(string State, string Message, int Percentage)> GetExport(long id) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/migration/exports/{id}"; + + var content = await _client.GetAsync(url); + var data = JObject.Parse(content); + + return ( + (string)data["state"], + (string)data["progress"]["message"], + (int)data["progress"]["percentage"] + ); + } + + public virtual async Task> GetProjects() + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects"; + return await _client.GetAllAsync(url) + .Select(x => ((int)x["id"], (string)x["key"], (string)x["name"])) + .ToListAsync(); + } + + public virtual async Task<(int Id, string Key, string Name)> GetProject(string projectKey) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}"; + var response = await _client.GetAsync(url); + + var project = JObject.Parse(response); + return ((int)project["id"], (string)project["key"], (string)project["name"]); + } + + public virtual async Task> GetRepos(string projectKey) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos"; + return await _client.GetAllAsync(url) + .Select(x => ((int)x["id"], (string)x["slug"], (string)x["name"])) + .ToListAsync(); + } + + public virtual async Task GetIsRepositoryArchived(string projectKey, string repo) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}?fields=archived"; + var response = await _client.GetAsync(url); + + var data = JObject.Parse(response); + return (bool)data["archived"]; + } + + public virtual async Task> GetRepositoryPullRequests(string projectKey, string repo) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/pull-requests?state=all"; + return await _client.GetAllAsync(url) + .Select(x => ((int)x["id"], (string)x["name"])) + .ToListAsync(); + } + + public virtual async Task GetRepositoryLatestCommitDate(string projectKey, string repo) + { + var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/commits?limit=1"; + + try + { + var response = await _client.GetAsync(url); + var commit = JObject.Parse(response); + + if (commit?["values"] == null || !commit["values"].Any()) + { + return null; + } + + var authorTimestamp = (long)commit["values"][0]["authorTimestamp"]; + return DateTimeOffset.FromUnixTimeMilliseconds(authorTimestamp).DateTime; + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public virtual async Task<(ulong repoSize, ulong attachmentsSize)> GetRepositoryAndAttachmentsSize(string projectKey, string repo, string gitlabUsername, string gitlabPassword) + { + var url = $"{_gitlabBaseUrl}/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/sizes"; + var response = await _client.GetAsync(url); + + var data = JObject.Parse(response); + + var repoSize = (ulong)data["repository"]; + var attachmentsSize = (ulong)data["attachments"]; + + return (repoSize, attachmentsSize); + } +} diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs new file mode 100644 index 000000000..7b7d6bb1f --- /dev/null +++ b/src/Octoshift/Services/GitlabClient.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; +using System.Web; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabClient +{ + private const int DEFAULT_PAGE_SIZE = 100; + private readonly HttpClient _httpClient; + private readonly OctoLogger _log; + private readonly RetryPolicy _retryPolicy; + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string username, string password) : + this(log, httpClient, versionProvider, retryPolicy) + { + if (_httpClient != null) + { + var authCredentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")); + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", authCredentials); + } + } + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy) + { + _log = log; + _httpClient = httpClient; + _retryPolicy = retryPolicy; + + if (_httpClient != null) + { + _httpClient.DefaultRequestHeaders.Add("accept", "application/json"); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); + if (versionProvider?.GetVersionComments() is { } comments) + { + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(comments)); + } + } + } + + public virtual async Task GetAsync(string url) + { + return await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + } + + public virtual async IAsyncEnumerable GetAllAsync(string url) + { + var hasNextPage = true; + var nextPageStart = 0; + while (hasNextPage) + { + var response = await GetWithPagination(url, nextPageStart); + var jResponse = JObject.Parse(response); + + foreach (var jToken in jResponse["values"]!) + { + yield return jToken; + } + + hasNextPage = !jResponse["isLastPage"]?.ToObject() ?? false; + nextPageStart = jResponse["nextPageStart"]?.ToObject() ?? 0; + } + } + + public virtual async Task PostAsync(string url, object body) => await SendAsync(HttpMethod.Post, url, body); + + public virtual async Task DeleteAsync(string url) => await SendAsync(HttpMethod.Delete, url); + + private async Task GetWithPagination(string url, int start = 0, int limit = DEFAULT_PAGE_SIZE) => await GetAsync(AddPaginationParams(url, start, limit)); + + private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) + { + _log.LogVerbose($"HTTP {httpMethod}: {url}"); + + if (body != null) + { + _log.LogVerbose($"HTTP BODY: {body.ToJson()}"); + } + + using var payload = body?.ToJson().ToStringContent(); + var response = httpMethod.ToString() switch + { + "GET" => await _httpClient.GetAsync(url), + "DELETE" => await _httpClient.DeleteAsync(url), + "POST" => await _httpClient.PostAsync(url, payload), + "PUT" => await _httpClient.PutAsync(url, payload), + "PATCH" => await _httpClient.PatchAsync(url, payload), + _ => throw new ArgumentOutOfRangeException($"{httpMethod} is not supported.") + }; + var content = await response.Content.ReadAsStringAsync(); + _log.LogVerbose($"RESPONSE ({response.StatusCode}): {content}"); + + response.EnsureSuccessStatusCode(); + + return content; + } + + private string AddPaginationParams(string url, int start, int limit) + { + var uri = new Uri(url); + var path = uri.GetLeftPart(UriPartial.Path); + var queryParams = HttpUtility.ParseQueryString(uri.Query); + + queryParams["start"] = start.ToString(); + queryParams["limit"] = limit.ToString(); + + return $"{path}?{queryParams}"; + } +} diff --git a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj index 833ba590f..835eeefa4 100644 --- a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj +++ b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs new file mode 100644 index 000000000..1464e8562 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.AbortMigration; + +public class AbortMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new AbortMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("abort-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs new file mode 100644 index 000000000..a9a32e501 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.CreateTeam; + +public class CreateTeamCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new CreateTeamCommand(); + Assert.NotNull(command); + Assert.Equal("create-team", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "team-name", true); + TestHelpers.VerifyCommandOption(command.Options, "idp-group", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs new file mode 100644 index 000000000..6c2a3073a --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs @@ -0,0 +1,38 @@ +using System.Linq; +using OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DownloadLogs; + +public class DownloadLogsCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new DownloadLogsCommand(); + Assert.NotNull(command); + Assert.Equal("download-logs", command.Name); + Assert.Equal(8, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-log-file", false); + TestHelpers.VerifyCommandOption(command.Options, "overwrite", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } + + [Fact] + public void Should_Support_Github_Api_Url_Alias_For_Backward_Compatibility() + { + // Test that --github-api-url still works as an alias for --target-api-url + var command = new DownloadLogsCommand(); + var option = command.Options.FirstOrDefault(o => o.Name == "target-api-url"); + + Assert.NotNull(option); + Assert.Contains("--target-api-url", option.Aliases); + Assert.Contains("--github-api-url", option.Aliases); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs new file mode 100644 index 000000000..c7b663412 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateMannequinCsv; + +public class GenerateMannequinCsvCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GenerateMannequinCsvCommand(); + Assert.NotNull(command); + Assert.Equal("generate-mannequin-csv", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "output", false); + TestHelpers.VerifyCommandOption(command.Options, "include-reclaimed", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs new file mode 100644 index 000000000..717bb0b17 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -0,0 +1,65 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgsTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly GenerateScriptCommandArgs _args = new(); + + [Fact] + public void It_Throws_If_GitlabServer_Url_Is_Not_Provided_But_No_Ssl_Verify_Is_Provided() + { + // Act + _args.NoSslVerify = true; + _args.GitlabServerUrl = ""; + + // Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--bbs-server-url*"); + } + + [Fact] + public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() + { + _args.SshPort = 7999; + + _args.Validate(_mockOctoLogger.Object); + + _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); + } + + [Fact] + public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsBucketName = "my-bucket"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + [Fact] + public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsRegion = "aws-region"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs new file mode 100644 index 000000000..735758eaf --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -0,0 +1,839 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandlerTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + + private readonly GenerateScriptCommandHandler _handler; + + private const string GITHUB_ORG = "GITHUB-ORG"; + private const string BBS_SERVER_URL = "http://bbs-server-url"; + private const string BBS_USERNAME = "BBS-USERNAME"; + private const string BBS_PASSWORD = "BBS-PASSWORD"; + private const string SSH_USER = "SSH-USER"; + private const string SSH_PRIVATE_KEY = "path-to-ssh-private-key"; + private const string ARCHIVE_DOWNLOAD_HOST = "archive-download-host"; + private const int SSH_PORT = 2211; + private const string SMB_USER = "SMB-USER"; + private const string SMB_DOMAIN = "SMB-DOMAIN"; + private const string OUTPUT = "unit-test-output"; + private const string BBS_FOO_PROJECT_KEY = "FP"; + private const string BBS_FOO_PROJECT_NAME = "BBS-FOO-PROJECT-NAME"; + private const string BBS_BAR_PROJECT_KEY = "BBS-BAR-PROJECT-NAME"; + private const string BBS_BAR_PROJECT_NAME = "BP"; + private const string BBS_FOO_REPO_1_SLUG = "foorepo1"; + private const string BBS_FOO_REPO_1_NAME = "BBS-FOO-REPO-1-NAME"; + private const string BBS_FOO_REPO_2_SLUG = "foorepo2"; + private const string BBS_FOO_REPO_2_NAME = "BBS-FOO-REPO-2-NAME"; + private const string BBS_BAR_REPO_1_SLUG = "barrepo1"; + private const string BBS_BAR_REPO_1_NAME = "BBS-BAR-REPO-1-NAME"; + private const string BBS_BAR_REPO_2_SLUG = "barrepo2"; + private const string BBS_BAR_REPO_2_NAME = "BBS-BAR-REPO-2-NAME"; + private const string BBS_SHARED_HOME = "BBS-SHARED-HOME"; + private const string AWS_BUCKET_NAME = "AWS-BUCKET-NAME"; + private const string AWS_REGION = "AWS_REGION"; + private const string UPLOADS_URL = "UPLOADS-URL"; + + public GenerateScriptCommandHandlerTests() + { + _handler = new GenerateScriptCommandHandler( + _mockOctoLogger.Object, + _mockVersionProvider.Object, + _mockFileSystemProvider.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object); + + _mockEnvironmentVariableProvider.Setup(m => m.GitlabUsername(It.IsAny())).Returns(BBS_USERNAME); + _mockEnvironmentVariableProvider.Setup(m => m.GitlabPassword(It.IsAny())).Returns(BBS_PASSWORD); + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] { (1, BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT_NAME) }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] { (1, BBS_FOO_REPO_1_SLUG, BBS_FOO_REPO_1_NAME) }); + } + + [Fact] + public async Task No_Projects() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT) + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 33, 0) == ""))); + } + + [Fact] + public async Task Validates_Env_Vars() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:GH_PAT) { + Write-Error ""GH_PAT environment variable must be set to a valid GitHub Personal Access Token with the appropriate scopes. For more information see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#creating-a-personal-access-token-for-github-enterprise-importer"" + exit 1 +} else { + Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" +} + +if (-not $env:BBS_PASSWORD) { + Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" +} + +if (-not $env:BBS_USERNAME) { + Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" +} + +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 9, 0) == TrimNonExecutableLines(expected, 0, 0)))); + } + + [Fact] + public async Task Validates_Env_Vars_BBS_USERNAME_Not_Validated_When_Passed_As_Arg() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + GitlabUsername = BBS_USERNAME, + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:BBS_USERNAME) { + Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call BBS API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to BBS APIs."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); + } + + [Fact] + public async Task Validates_Env_Vars_BBS_PASSWORD_Not_Validated_When_Kerberos() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + Kerberos = true, + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:BBS_PASSWORD) { + Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call BBS API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to BBS APIs."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); + } + + [Fact] + public async Task Validates_Env_Vars_AWS() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + AwsBucketName = AWS_BUCKET_NAME, + Output = new FileInfo(OUTPUT), + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +} +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); + } + + [Fact] + public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_Not_Validated_When_Aws() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + AwsBucketName = AWS_BUCKET_NAME, + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); + } + + [Fact] + public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_And_AWS_Not_Validated_When_UseGithubStorage() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true + }; + await _handler.Handle(args); + + var expectedAws = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +} +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + + var expectedAzure = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAws, 0, 0))))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAzure, 0, 0))))); + } + + [Fact] + public async Task Validates_Env_Vars_SMB_PASSWORD() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT), + SmbUser = SMB_USER, + }; + await _handler.Handle(args); + + var expected = @" +if (-not $env:SMB_PASSWORD) { + Write-Error ""SMB_PASSWORD environment variable must be set to a valid password that will be used to download the migration archive from your BBS server using SMB."" + exit 1 +} else { + Write-Host ""SMB_PASSWORD environment variable is set and will be used to download the migration archive from your BBS server using SMB."" +}"; + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); + } + + [Fact] + public async Task No_Repos() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(Enumerable.Empty<(int Id, string Slug, string Name)>()); + + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT) + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 33, 0) == ""))); + } + + [Fact] + public async Task Two_Projects_Two_Repos_Each_All_Options() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + (Id: 2, Key: BBS_BAR_PROJECT_KEY, Name: BBS_BAR_PROJECT_NAME) + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + (Id: 2, Slug: BBS_FOO_REPO_2_SLUG, Name: BBS_FOO_REPO_2_NAME) + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_BAR_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 3, Slug: BBS_BAR_REPO_1_SLUG, Name: BBS_BAR_REPO_1_NAME), + (Id: 4, Slug: BBS_BAR_REPO_2_SLUG, Name: BBS_BAR_REPO_2_NAME) + }); + + var migrateRepoCommand1 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; + var migrateRepoCommand2 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_2_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; + var migrateRepoCommand3 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_BAR_PROJECT_KEY}\" --bbs-repo \"{BBS_BAR_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_BAR_PROJECT_KEY}-{BBS_BAR_REPO_1_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; + var migrateRepoCommand4 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_BAR_PROJECT_KEY}\" --bbs-repo \"{BBS_BAR_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_BAR_PROJECT_KEY}-{BBS_BAR_REPO_2_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + Output = new FileInfo(OUTPUT), + Verbose = true, + KeepArchive = true + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand1)))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand2)))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand3)))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand4)))); + + _mockEnvironmentVariableProvider.Verify(m => m.GitlabUsername(It.IsAny()), Times.Never); + _mockEnvironmentVariableProvider.Verify(m => m.GitlabPassword(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Filters_By_Project() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + (Id: 2, Key: BBS_BAR_PROJECT_KEY, Name: BBS_BAR_PROJECT_NAME) + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + (Id: 2, Slug: BBS_FOO_REPO_2_SLUG, Name: BBS_FOO_REPO_2_NAME) + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_BAR_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 3, Slug: BBS_BAR_REPO_1_SLUG, Name: BBS_BAR_REPO_1_NAME), + (Id: 4, Slug: BBS_BAR_REPO_2_SLUG, Name: BBS_BAR_REPO_2_NAME) + }); + + var migrateRepoCommand1 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; + var migrateRepoCommand2 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_2_SLUG}\" --verbose --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_FOO_PROJECT_KEY, + GitlabSharedHome = BBS_SHARED_HOME, + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + Output = new FileInfo(OUTPUT), + Verbose = true + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand1)))); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand2)))); + + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(BBS_BAR_PROJECT_KEY))), Times.Never); + } + + [Fact] + public async Task One_Repo_With_Kerberos() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --kerberos --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + Output = new FileInfo(OUTPUT), + Verbose = true, + Kerberos = true, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task One_Repo_With_No_Ssl_Verify() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --no-ssl-verify --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + Output = new FileInfo(OUTPUT), + Verbose = true, + NoSslVerify = true, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task One_Repo_With_Smb() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + SmbUser = SMB_USER, + SmbDomain = SMB_DOMAIN, + Output = new FileInfo(OUTPUT), + Verbose = true + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task One_Repo_With_Smb_And_TargetApiUrl() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + var targetApiUrl = "https://foo.com/api/v3"; + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --target-api-url \"{targetApiUrl}\" --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + SmbUser = SMB_USER, + SmbDomain = SMB_DOMAIN, + Output = new FileInfo(OUTPUT), + Verbose = true, + TargetApiUrl = targetApiUrl + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task One_Repo_With_Smb_And_Archive_Download_Host() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + SmbUser = SMB_USER, + SmbDomain = SMB_DOMAIN, + Output = new FileInfo(OUTPUT), + Verbose = true + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task Generated_Script_Contains_The_Cli_Version_Comment() + { + // Arrange + _mockVersionProvider.Setup(m => m.GetCurrentVersion()).Returns("1.1.1"); + const string cliVersionComment = "# =========== Created with CLI version 1.1.1 ==========="; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT) + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(cliVersionComment)))); + } + + [Fact] + public async Task Generated_Script_StartsWith_Shebang() + { + // Arrange + const string shebang = "#!/usr/bin/env pwsh"; + + // Act + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT) + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.StartsWith(shebang)))); + } + + [Fact] + public async Task Generated_Script_Contains_Exec_Function_Block() + { + // Arrange + const string execFunctionBlock = @" +function Exec { + param ( + [scriptblock]$ScriptBlock + ) + & @ScriptBlock + if ($lastexitcode -ne 0) { + exit $lastexitcode + } +}"; + + var args = new GenerateScriptCommandArgs() + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + Output = new FileInfo(OUTPUT) + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(execFunctionBlock)))); + } + + [Fact] + public async Task One_Repo_With_Aws_Bucket_Name_And_Region() + { + // Arrange + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), + }); + + var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" " + + $"--bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" " + + $"--ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" " + + $"--github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --aws-bucket-name \"{AWS_BUCKET_NAME}\" " + + $"--aws-region \"{AWS_REGION}\" --target-repo-visibility private }}"; + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabSharedHome = BBS_SHARED_HOME, + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + Output = new FileInfo(OUTPUT), + Verbose = true, + AwsBucketName = AWS_BUCKET_NAME, + AwsRegion = AWS_REGION + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + } + + [Fact] + public async Task BBS_Single_Repo_With_UseGithubStorage() + { + // Arrange + var TARGET_API_URL = "https://foo.com/api/v3"; + const string BBS_PROJECT_KEY = "BBS-PROJECT"; + const string BBS_REPO_SLUG = "repo-slug"; + + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), + }); + + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo("unit-test-output"), + UseGithubStorage = true, + TargetApiUrl = TARGET_API_URL, + GitlabProject = BBS_PROJECT_KEY, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => + script.Contains("--bbs-server-url \"http://bbs-server-url\"") && + script.Contains("--bbs-project \"BBS-PROJECT\"") && + script.Contains("--github-org \"GITHUB-ORG\"") && + script.Contains("--use-github-storage") +))); + + } + [Fact] + public async Task BBS_Single_Repo_With_TargetUploadsUrl() + { + // Arrange + var TARGET_API_URL = "https://foo.com/api/v3"; + const string BBS_PROJECT_KEY = "BBS-PROJECT"; + const string BBS_REPO_SLUG = "repo-slug"; + + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] + { + (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), + }); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] + { + (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), + }); + + // Act + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo("unit-test-output"), + TargetApiUrl = TARGET_API_URL, + TargetUploadsUrl = UPLOADS_URL, + GitlabProject = BBS_PROJECT_KEY, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => + script.Contains("--bbs-server-url \"http://bbs-server-url\"") && + script.Contains("--bbs-project \"BBS-PROJECT\"") && + script.Contains("--github-org \"GITHUB-ORG\"") && + script.Contains("--target-uploads-url \"UPLOADS-URL\"") + ))); + } + + private string TrimNonExecutableLines(string script, int skipFirst = 9, int skipLast = 0) + { + var lines = script.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).AsEnumerable(); + + lines = lines + .Where(x => x.HasValue()) + .Where(x => !x.Trim().StartsWith("#")) + .Skip(skipFirst) + .SkipLast(skipLast); + + var result = string.Join(Environment.NewLine, lines); + return result; + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs new file mode 100644 index 000000000..788531326 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -0,0 +1,129 @@ +using System; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandTests +{ + private const string BBS_SERVER_URL = "http://bbs.contoso.com:7990"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + + private readonly GenerateScriptCommand _command = []; + + public GenerateScriptCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IVersionProvider))).Returns(_mockVersionProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + } + + [Fact] + public void Should_Have_Options() + { + _command.Should().NotBeNull(); + _command.Name.Should().Be("generate-script"); + _command.Options.Count.Should().Be(21); + + TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-username", false); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-password", false); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-project", false); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-shared-home", false); + TestHelpers.VerifyCommandOption(_command.Options, "archive-download-host", false); + TestHelpers.VerifyCommandOption(_command.Options, "ssh-user", false); + TestHelpers.VerifyCommandOption(_command.Options, "ssh-private-key", false); + TestHelpers.VerifyCommandOption(_command.Options, "ssh-port", false); + TestHelpers.VerifyCommandOption(_command.Options, "smb-user", false); + TestHelpers.VerifyCommandOption(_command.Options, "smb-domain", false); + TestHelpers.VerifyCommandOption(_command.Options, "output", false); + TestHelpers.VerifyCommandOption(_command.Options, "kerberos", false, true); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); + } + + [Fact] + public void It_Gets_A_Kerberos_HttpClient_When_Kerberos_Is_True() + { + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + Kerberos = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, false)); + } + + [Fact] + public void It_Gets_A_Kerberos_With_No_Ssl_Verify_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_True() + { + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + Kerberos = true, + NoSslVerify = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, true)); + } + + [Fact] + public void It_Gets_A_Default_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_Not_Set() + { + var bbsTestUser = "user"; + var bbsTestPassword = "password"; + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = bbsTestUser, + GitlabPassword = bbsTestPassword + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, false)); + } + + [Fact] + public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_Set() + { + var bbsTestUser = "user"; + var bbsTestPassword = "password"; + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = bbsTestUser, + GitlabPassword = bbsTestPassword, + NoSslVerify = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, true)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs new file mode 100644 index 000000000..f61e6e528 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GrantMigratorRole; + +public class GrantMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GrantMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("grant-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs new file mode 100644 index 000000000..fa398cc1e --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs @@ -0,0 +1,132 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandlerTests +{ + private const string BBS_SERVER_URL = "http://bbs-server-url"; + private const string BBS_PROJECT_KEY = "FP"; + private const string BBS_PROJECT = "foo-project"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const bool NO_SSL_VERIFY = true; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGenerator = TestHelpers.CreateMock(); + private readonly Mock _mockReposCsvGenerator = TestHelpers.CreateMock(); + + private string _projectsCsvOutput = ""; + private string _reposCsvOutput = ""; + + private readonly InventoryReportCommandHandler _handler; + + public InventoryReportCommandHandlerTests() + { + _handler = new InventoryReportCommandHandler( + TestHelpers.CreateMock().Object, + _mockGitlabApi.Object, + _mockGitlabInspectorService.Object, + _mockProjectsCsvGenerator.Object, + _mockReposCsvGenerator.Object) + { + WriteToFile = (path, contents) => + { + if (path == "projects.csv") + { + _projectsCsvOutput = contents; + } + + if (path == "repos.csv") + { + _reposCsvOutput = contents; + } + + return Task.CompletedTask; + } + }; + } + + [Fact] + public async Task Happy_Path() + { + var expectedProjectsCsv = "csv stuff"; + var expectedReposCsv = "repo csv stuff"; + + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] { (Id: 1, Key: BBS_PROJECT_KEY, Name: BBS_PROJECT) }); + _mockGitlabInspectorService.Setup(m => m.GetRepoCount()).ReturnsAsync(1); + + _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedProjectsCsv); + _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedReposCsv); + + // var args = new InventoryReportCommandArgs(); + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + _reposCsvOutput.Should().Be(expectedReposCsv); + } + + [Fact] + public async Task Scoped_To_Single_Project() + { + var expectedProjectsCsv = "csv stuff"; + var expectedReposCsv = "repo csv stuff"; + + _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_PROJECT, false)).ReturnsAsync(expectedProjectsCsv); + _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_PROJECT, false)).ReturnsAsync(expectedReposCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + _reposCsvOutput.Should().Be(expectedReposCsv); + } + + [Fact] + public async Task It_Generates_Minimal_Csvs_When_Requested() + { + // Arrange + var expectedProjectsCsv = "csv stuff"; + var expectedReposCsv = "repo csv stuff"; + + _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedProjectsCsv); + _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedReposCsv); + + // Act + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + NoSslVerify = NO_SSL_VERIFY, + Minimal = true + }; + await _handler.Handle(args); + + // Assert + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + _reposCsvOutput.Should().Be(expectedReposCsv); + + _mockProjectsCsvGenerator.Verify(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, true)); + _mockReposCsvGenerator.Verify(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, true)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs new file mode 100644 index 000000000..2e94f7be5 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.DependencyInjection; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandTests + { + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGeneratorService = TestHelpers.CreateMock(); + private readonly Mock _mockReposCsvGeneratorService = TestHelpers.CreateMock(); + + private readonly ServiceProvider _serviceProvider; + private readonly InventoryReportCommand _command = []; + + public InventoryReportCommandTests() + { + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(_mockOctoLogger.Object) + .AddSingleton(_mockGitlabApi.Object) + .AddSingleton(_mockGitlabInspectorServiceFactory.Object) + .AddSingleton(_mockProjectsCsvGeneratorService.Object) + .AddSingleton(_mockReposCsvGeneratorService.Object); + + _serviceProvider = serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Should_Have_Options() + { + Assert.NotNull(_command); + Assert.Equal("inventory-report", _command.Name); + Assert.Equal(7, _command.Options.Count); + + TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-project", false); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-username", false); + TestHelpers.VerifyCommandOption(_command.Options, "bbs-password", false); + TestHelpers.VerifyCommandOption(_command.Options, "minimal", false); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs new file mode 100644 index 000000000..500bad660 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs @@ -0,0 +1,652 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo +{ + public class MigrateRepoCommandArgsTests + { + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + + private const string ARCHIVE_PATH = "path/to/archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/bbs-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string GITHUB_PAT = "github pat"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_SESSION_TOKEN = "aws-session-token"; + private const string AWS_REGION = "aws-region"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string BBS_HOST = "our-bbs-server.com"; + private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const string BBS_PROJECT = "bbs-project"; + private const string BBS_REPO = "bbs-repo"; + private const string SSH_USER = "ssh-user"; + private const string PRIVATE_KEY = "private-key"; + private const string SMB_USER = "smb-user"; + private const string SMB_PASSWORD = "smb-password"; + + [Fact] + public void It_Throws_When_Kerberos_Is_Set_And_Gitlab_Password_Is_Provided() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabPassword = BBS_PASSWORD, + Kerberos = true + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-password*--kerberos*"); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Access_Key_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsAccessKey = AWS_ACCESS_KEY_ID + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Provided_With_UseGithubStorage_Option() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsBucketName = AWS_BUCKET_NAME, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with an AWS S3 Bucket name*"); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Provided_With_AzureStorageConnectionString_Option() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsBucketName = AWS_BUCKET_NAME, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*Archive cannot be uploaded to both locations."); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Secret_Key_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSecretKey = AWS_SECRET_ACCESS_KEY + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Session_Token_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSessionToken = AWS_SESSION_TOKEN + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Region_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsRegion = AWS_REGION + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Project() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabRepo = BBS_REPO, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-project*"); + } + + [Fact] + public void Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Repo() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-repo*"); + } + + [Fact] + public void It_Throws_When_Kerberos_Is_Set_And_Gitlab_Username_Is_Provided() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabUsername = BBS_USERNAME, + Kerberos = true + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-username*--kerberos*"); + } + + [Fact] + public void Errors_If_Gitlab_Password_Is_Provided_With_Archive_Path() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GitlabPassword = BBS_USERNAME + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-username*--bbs-password*--archive-path*"); + } + + [Fact] + public void Errors_If_Gitlab_Password_Is_Provided_With_Archive_Url() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GitlabPassword = BBS_USERNAME + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-username*--bbs-password*--archive-url*"); + } + + [Fact] + public void Errors_If_No_Ssl_Verify_Is_Provided_With_Archive_Path() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + NoSslVerify = true + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--archive-path*"); + } + + [Fact] + public void Errors_If_No_Ssl_Verify_Is_Provided_With_Archive_Url() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + NoSslVerify = true + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--archive-url*"); + } + + [Fact] + public void Errors_If_Ssh_User_Is_Provided_With_Archive_Path() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*SSH*SMB*--archive-path*"); + } + + [Fact] + public void Errors_If_Ssh_User_Is_Provided_With_Archive_Url() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*SSH*SMB*--archive-url*"); + } + + [Fact] + public void Errors_If_Smb_User_Is_Provided_With_Archive_Path() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + SmbUser = SMB_USER, + SmbPassword = SMB_PASSWORD, + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*SSH*SMB*--archive-path*"); + } + + [Fact] + public void Errors_If_Smb_User_Is_Provided_With_Archive_Url() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + SmbUser = SMB_USER, + SmbPassword = SMB_PASSWORD, + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*SSH*SMB*--archive-url*"); + } + + [Fact] + public void It_Throws_If_Github_Org_Is_Provided_But_Github_Repo_Is_Not() + { + // Act + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GitlabServerUrl = BBS_SERVER_URL, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GithubPat = GITHUB_PAT, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-repo*"); + } + + [Fact] + public void It_Throws_If_Archive_Url_Is_Provided_But_Github_Org_Is_Not() + { + // Act + var args = new MigrateRepoCommandArgs + { + GithubPat = GITHUB_PAT, + ArchiveUrl = ARCHIVE_URL, + GithubRepo = GITHUB_REPO + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-org*"); + } + + [Fact] + public void It_Throws_If_Archive_Url_Is_Provided_But_Github_Repo_Is_Not() + { + // Act + var args = new MigrateRepoCommandArgs + { + GithubPat = GITHUB_PAT, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-repo*"); + } + + [Fact] + public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_User_And_Smb_User_Are_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SmbUser = SMB_USER + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Errors_When_Archive_Download_Host_Provided_Without_Ssh_Or_Smb_Options() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + ArchiveDownloadHost = "somehost" + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_User_And_Smb_Password_Are_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SmbPassword = SMB_PASSWORD + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_Private_Key_And_Smb_User_Are_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshPrivateKey = PRIVATE_KEY, + SmbUser = SMB_USER + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_Private_Key_And_Smb_Password_Are_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + SmbPassword = SMB_PASSWORD + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Invoke_With_Gitlab_Server_Url_Throws_When_Ssh_User_Is_Provided_And_Ssh_Private_Key_Is_Not_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); + } + + [Fact] + public void Errors_If_Archive_Url_And_Archive_Path_Are_Passed() + { + // Act + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--archive-path*--archive-url*"); + } + + [Fact] + public void Allows_GitlabServer_Url_And_Archive_Url_To_Be_Passed_Together() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .NotThrow(); + } + + [Fact] + public void Allows_GitlabServer_Url_And_Archive_Path_To_Be_Passed_Together() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .NotThrow(); + } + + [Fact] + public void Errors_If_GitlabServer_Url_Archive_Path_And_Archive_Url_Are_Not_Provided() + { + // Act + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + // Assert + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--bbs-server-url*--archive-path*--archive-url*"); + } + + [Fact] + public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() + { + var args = new MigrateRepoCommandArgs + { + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + SshPort = 7999 + }; + + args.Validate(_mockOctoLogger.Object); + + _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs new file mode 100644 index 000000000..74ef78b0b --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs @@ -0,0 +1,1079 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo +{ + public class MigrateRepoCommandHandlerTests + { + private readonly Mock _mockGithubApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApi = TestHelpers.CreateMock(); + private readonly Mock _mockAwsApi = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabArchiveDownloader = new(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + + private readonly WarningsCountLogger _warningsCountLogger; + private readonly MigrateRepoCommandHandler _handler; + + private const string ARCHIVE_PATH = "path/to/archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/bbs-archive.tar"; + private readonly byte[] ARCHIVE_DATA = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string GITHUB_PAT = "github pat"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_SESSION_TOKEN = "aws-session-token"; + private const string AWS_REGION = "eu-west-1"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + + private const string BBS_HOST = "our-bbs-server.com"; + private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const string BBS_PROJECT = "bbs-project"; + private const string BBS_REPO = "bbs-repo"; + private const string BBS_REPO_URL = $"{BBS_SERVER_URL}/projects/{BBS_PROJECT}/repos/{BBS_REPO}/browse"; + private const string UNUSED_REPO_URL = "https://not-used"; + private const string SSH_USER = "ssh-user"; + private const string PRIVATE_KEY = "private-key"; + private const string SMB_USER = "smb-user"; + private const string SMB_PASSWORD = "smb-password"; + private const long BBS_EXPORT_ID = 123; + + private const string GITHUB_ORG_ID = "github-org-id"; + private const string MIGRATION_SOURCE_ID = "migration-source-id"; + private const string MIGRATION_ID = "migration-id"; + + public MigrateRepoCommandHandlerTests() + { + _warningsCountLogger = new WarningsCountLogger(_mockOctoLogger.Object); + _handler = new MigrateRepoCommandHandler( + _mockOctoLogger.Object, + _mockGithubApi.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object, + _mockGitlabArchiveDownloader.Object, + _mockAzureApi.Object, + _mockAwsApi.Object, + _mockFileSystemProvider.Object, + _warningsCountLogger + ); + + // Default setup for file system operations + _mockFileSystemProvider.Setup(m => m.FileExists(It.IsAny())).Returns(true); + _mockFileSystemProvider.Setup(m => m.DirectoryExists(It.IsAny())).Returns(true); + } + + [Fact] + public async Task Happy_Path_Generate_Only() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + }; + await _handler.Handle(args); + + // Assert + _mockGitlabApi.Verify(m => m.StartExport( + BBS_PROJECT, + BBS_REPO + )); + + _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Happy_Path_Generate_And_Download() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + }; + await _handler.Handle(args); + + // Assert + _mockGitlabArchiveDownloader.Verify(m => m.Download(BBS_EXPORT_ID, It.IsAny())); + _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Happy_Path_Ingest_Only() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Happy_Path_Generate_Archive_Ssh_Download_Azure_Upload_And_Ingest() + { + // Arrange + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + BBS_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Happy_Path_Generate_Archive_Ssh_Download_Aws_Upload_And_Ingest() + { + // Arrange + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsRegion = AWS_REGION, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + BBS_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Happy_Path_Full_Flow_Running_On_Gitlab_Server() + { + // Arrange + const string bbsSharedHome = "bbs-shared-home"; + var archivePath = $"{bbsSharedHome}/data/migration/export/Bitbucket_export_{BBS_EXPORT_ID}.tar"; + + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + GitlabSharedHome = bbsSharedHome, + QueueOnly = true, + }; + + var handler = new MigrateRepoCommandHandler( + _mockOctoLogger.Object, + _mockGithubApi.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object, + null, // in case of running on Bitbucket server, the downloader will be null + _mockAzureApi.Object, + _mockAwsApi.Object, + _mockFileSystemProvider.Object, + _warningsCountLogger + ); + await handler.Handle(args); + + // Assert + args.ArchivePath.Should().Be(archivePath); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + BBS_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Happy_Path_Full_Flow_Gitlab_Credentials_Via_Environment() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.GitlabUsername(It.IsAny())).Returns(BBS_USERNAME); + _mockEnvironmentVariableProvider.Setup(m => m.GitlabPassword(It.IsAny())).Returns(BBS_PASSWORD); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + BBS_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Happy_Path_Uploads_To_Github_Storage() + { + // Arrange + var githubOrgDatabaseId = Guid.NewGuid().ToString(); + const string gitArchiveFilePath = "./gitdata_archive"; + const string gitArchiveUrl = "gei://archive/1"; + const string gitArchiveContents = "I am git archive"; + + await using var gitContentStream = new MemoryStream(gitArchiveContents.ToBytes()); + + _mockFileSystemProvider.Setup(m => m.OpenRead(gitArchiveFilePath)).Returns(gitContentStream); + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.GetOrganizationDatabaseId(GITHUB_ORG).Result).Returns(githubOrgDatabaseId); + _mockGithubApi + .Setup(x => x.UploadArchiveToGithubStorage( + githubOrgDatabaseId, + It.IsAny(), + It.Is(s => (s as MemoryStream).ToArray().GetString() == gitArchiveContents)).Result) + .Returns(gitArchiveUrl); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + ArchivePath = gitArchiveFilePath, + UseGithubStorage = true, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + BBS_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + gitArchiveUrl, + null + )); + } + + [Fact] + public async Task Happy_Path_Deletes_Downloaded_Archive() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); + } + + [Fact] + public async Task It_Deletes_Downloaded_Archive_Even_If_Upload_Fails() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ThrowsAsync(new InvalidOperationException()); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT + }; + await _handler.Invoking(async x => await x.Handle(args)).Should().ThrowExactlyAsync(); + + // Assert + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); + } + + [Fact] + public async Task Happy_Path_Does_Not_Throw_If_Fails_To_Delete_Downloaded_Archive() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(x => x.DeleteIfExists(It.IsAny())).Throws(new UnauthorizedAccessException("Access Denied")); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockFileSystemProvider.Verify(x => x.DeleteIfExists(ARCHIVE_PATH)); + } + + [Fact] + public async Task Dont_Generate_Archive_If_Target_Repo_Exists() + { + // Arrange + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(true); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await FluentActions + .Invoking(async () => await _handler.Handle(args)).Should().ThrowExactlyAsync(); + + // Assert + _mockGitlabApi.Verify(x => x.StartExport(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Uses_GitHub_Pat_When_Provided_As_Option() + { + // Arrange + var githubPat = "specific github pat"; + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + _mockGithubApi + .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) + .Returns(MIGRATION_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = githubPat, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + githubPat, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Skip_Migration_If_Target_Repo_Exists() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + _mockGithubApi + .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) + .Throws(new OctoshiftCliException($"A repository called {GITHUB_ORG}/{GITHUB_REPO} already exists")); + + // Act + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockOctoLogger.Verify(m => m.LogWarning(It.IsAny()), Times.Exactly(1)); + } + + [Fact] + public async Task Throws_Decorated_Error_When_Create_Migration_Source_Fails_With_Permissions_Error() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi + .Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result) + .Throws(new OctoshiftCliException("monalisa does not have the correct permissions to execute `CreateMigrationSource`")); + + // Act + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + })) + .Should() + .ThrowAsync() + .WithMessage($"monalisa does not have the correct permissions to execute `CreateMigrationSource`. Please check that:\n (a) you are a member of the `{GITHUB_ORG}` organization,\n (b) you are an organization owner or you have been granted the migrator role and\n (c) your personal access token has the correct scopes.\nFor more information, see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer."); + } + + [Fact] + public async Task Throws_An_Error_If_Export_Fails() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("FAILED", "The export failed", 0)); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + }; + + // Assert + await _handler.Invoking(x => x.Handle(args)).Should().ThrowExactlyAsync(); + } + + [Fact] + public async Task Uses_Archive_Path_If_Provided() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + + var archiveBytes = Encoding.ASCII.GetBytes("here are some bytes"); + _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(archiveBytes); + + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + _mockGithubApi + .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) + .Returns(MIGRATION_ID); + + // Act + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + } + + [Fact] + public async Task Invoke_With_Gitlab_Server_Url_Throws_When_Smb_User_Is_Provided_And_Smb_Password_Is_Not_Provided() + { + // Act, Assert + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + SmbUser = SMB_USER + }; + + await _handler.Invoking(x => x.Handle(args)).Should().ThrowExactlyAsync(); + } + + [Fact] + public async Task Invoke_With_Gitlab_Server_Url_Should_Not_Throw_When_Smb_User_Is_Provided_And_Smb_Password_Is_Provided_Via_Environment_Variable() + { + // Arrange + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + + _mockEnvironmentVariableProvider.Setup(m => m.SmbPassword(It.IsAny())).Returns(SMB_PASSWORD); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + }; + + // Act, Assert + await _handler.Invoking(x => x.Handle(args)).Should().NotThrowAsync(); + } + + [Fact] + public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Path_Is_Provided() + { + // Arrange + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + + // Act + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + args.ArchivePath.Should().Be(ARCHIVE_PATH); + _mockFileSystemProvider.Verify(m => m.OpenRead(ARCHIVE_PATH)); + } + + [Fact] + public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Url_Is_Provided() + { + // Act + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + await _handler.Handle(args); + + // Assert + args.ArchivePath.Should().BeNull(); + _mockFileSystemProvider.Verify(m => m.ReadAllBytesAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Uses_Aws_If_Credentials_Are_Passed() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); + _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); + + // Act + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + ArchivePath = ARCHIVE_PATH, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsBucketName = AWS_BUCKET_NAME, + AwsRegion = AWS_REGION, + QueueOnly = true, + }; + + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null + )); + + _mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())); + } + + [Fact] + public async Task It_Throws_When_Both_Azure_Storage_Connection_String_And_Aws_Bucket_Name_Are_Not_Provided() + { + await _handler.Invoking(async x => await x.Handle( + new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + })) + .Should() + .ThrowAsync() + .WithMessage("*--azure-storage-connection-string*AZURE_STORAGE_CONNECTION_STRING*or*--aws-bucket-name*--aws-access-key*AWS_ACCESS_KEY_ID*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); + } + + [Fact] + public async Task It_Throws_When_Both_Azure_Storage_Connection_String_And_Aws_Bucket_Name_Are_Provided() + { + await _handler.Invoking(async x => await x.Handle( + new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsBucketName = AWS_BUCKET_NAME + })) + .Should() + .ThrowAsync() + .WithMessage("*--azure-storage-connection-string*AZURE_STORAGE_CONNECTION_STRING*and*--aws-bucket-name*--aws-access-key*AWS_ACCESS_KEY_ID*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); + } + + [Fact] + public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_But_No_Aws_Access_Key_Id() + { + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME + })) + .Should() + .ThrowAsync() + .WithMessage("*--aws-access-key*AWS_ACCESS_KEY_ID*"); + } + + [Fact] + public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Secret_Access_Key() + { + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID + })) + .Should() + .ThrowAsync() + .WithMessage("*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); + } + + [Fact] + public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Region() + { + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsSessionToken = AWS_SESSION_TOKEN + })) + .Should() + .ThrowAsync() + .WithMessage("Either --aws-region or AWS_REGION environment variable must be set."); + } + + [Fact] + public async Task Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Username() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + // Assert + await _handler.Invoking(x => x.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*BBS_USERNAME*--bbs-username*"); + } + + [Fact] + public async Task Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Password() + { + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GitlabUsername = BBS_USERNAME + }; + + // Assert + await _handler.Invoking(x => x.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*BBS_PASSWORD*--bbs-password*"); + } + + [Fact] + public async Task It_Should_Not_Validate_Gitlab_Username_And_Password_When_Kerberos_Is_Set() + { + // Arrange + _mockGitlabApi.Setup(x => x.GetExport(It.IsAny())).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + + // Act + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + Kerberos = true + }; + + // Assert + await _handler.Invoking(x => x.Handle(args)) + .Should() + .NotThrowAsync(); + } + + [Fact] + public async Task Sets_Target_Repo_Visibility() + { + // Arrange + var targetRepoVisibility = "public"; + + // Act + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + TargetRepoVisibility = targetRepoVisibility, + }; + await _handler.Handle(args); + + // Assert + _mockGithubApi.Verify(m => m.StartGitlabMigration( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + targetRepoVisibility + )); + } + + [Fact] + public async Task It_Throws_When_Archive_Path_Does_Not_Exist() + { + const string nonExistentArchivePath = "/path/to/nonexistent/archive.tar"; + _mockFileSystemProvider.Setup(m => m.FileExists(nonExistentArchivePath)).Returns(false); + + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + ArchivePath = nonExistentArchivePath, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + })) + .Should() + .ThrowAsync() + .WithMessage($"*--archive-path*{nonExistentArchivePath}*"); + } + + [Fact] + public async Task It_Throws_When_Gitlab_Shared_Home_Does_Not_Exist_When_Running_On_Bitbucket_Instance() + { + const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); + + await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabSharedHome = nonExistentGitlabSharedHome + })) + .Should() + .ThrowAsync() + .WithMessage($"*--bbs-shared-home*{nonExistentGitlabSharedHome}*"); + } + + [Fact] + public async Task It_Does_Not_Validate_Gitlab_Shared_Home_When_Using_Ssh() + { + const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabSharedHome = nonExistentGitlabSharedHome, + SshUser = SSH_USER, + SshPrivateKey = PRIVATE_KEY + }; + + await _handler.Invoking(x => x.Handle(args)) + .Should() + .NotThrowAsync(); + } + + [Fact] + public async Task It_Does_Not_Validate_Gitlab_Shared_Home_When_Using_Smb() + { + const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; + _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); + _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); + _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); + _mockEnvironmentVariableProvider.Setup(m => m.SmbPassword(It.IsAny())).Returns(SMB_PASSWORD); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + GitlabProject = BBS_PROJECT, + GitlabRepo = BBS_REPO, + GitlabSharedHome = nonExistentGitlabSharedHome, + SmbUser = SMB_USER + }; + + await _handler.Invoking(x => x.Handle(args)) + .Should() + .NotThrowAsync(); + } + + [Fact] + public async Task It_Logs_Archive_Path_Before_Upload() + { + _mockFileSystemProvider.Setup(m => m.FileExists(ARCHIVE_PATH)).Returns(true); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + + await _handler.Handle(args); + _mockOctoLogger.Verify(m => m.LogInformation($"Archive path: {ARCHIVE_PATH}"), Times.Once); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs new file mode 100644 index 000000000..98b2550a8 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs @@ -0,0 +1,357 @@ +using System; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandTests +{ + private const string ARCHIVE_DOWNLOAD_HOST = "archive-download-host"; + private const string SSH_USER = "ssh-user"; + private const string SSH_PRIVATE_KEY = "ssh-private-key"; + private const int SSH_PORT = 1234; + private const string BBS_SHARED_HOME = "shared-home"; + private const string BBS_HOST = "bbs-host"; + private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; + private const string GITHUB_ORG = "github-org"; + private const string GITHUB_PAT = "github-pat"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string SMB_USER = "smb-user"; + private const string SMB_PASSWORD = "smb-password"; + private const string SMB_DOMAIN = "smb-domain"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGithubApiFactory = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabArchiveDownloaderFactory = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApiFactory = new(); + private readonly Mock _warningsCountLogger = TestHelpers.CreateMock(); + + private readonly MigrateRepoCommand _command = []; + + public MigrateRepoCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(ITargetGithubApiFactory))).Returns(_mockGithubApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabArchiveDownloaderFactory))).Returns(_mockGitlabArchiveDownloaderFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IAzureApiFactory))).Returns(_mockAzureApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(WarningsCountLogger))).Returns(_warningsCountLogger.Object); + } + + [Fact] + public void Should_Have_Options() + { + var command = new MigrateRepoCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("migrate-repo"); + command.Options.Count.Should().Be(33); + + TestHelpers.VerifyCommandOption(command.Options, "bbs-server-url", true); + TestHelpers.VerifyCommandOption(command.Options, "bbs-project", true); + TestHelpers.VerifyCommandOption(command.Options, "bbs-repo", true); + TestHelpers.VerifyCommandOption(command.Options, "bbs-username", false); + TestHelpers.VerifyCommandOption(command.Options, "bbs-password", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-url", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-path", false); + TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-session-token", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-secret-key", false); + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-download-host", false); + TestHelpers.VerifyCommandOption(command.Options, "ssh-user", false); + TestHelpers.VerifyCommandOption(command.Options, "ssh-private-key", false); + TestHelpers.VerifyCommandOption(command.Options, "ssh-port", false); + TestHelpers.VerifyCommandOption(command.Options, "smb-user", false); + TestHelpers.VerifyCommandOption(command.Options, "smb-password", false); + TestHelpers.VerifyCommandOption(command.Options, "smb-domain", false); + TestHelpers.VerifyCommandOption(command.Options, "queue-only", false); + TestHelpers.VerifyCommandOption(command.Options, "target-repo-visibility", false); + TestHelpers.VerifyCommandOption(command.Options, "kerberos", false, true); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Ssh_Archive_Downloader_Based_On_Server_Url_When_Ssh_User_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + GitlabSharedHome = BBS_SHARED_HOME, + GitlabServerUrl = BBS_SERVER_URL + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(BBS_HOST, SSH_USER, SSH_PRIVATE_KEY, SSH_PORT, BBS_SHARED_HOME)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Ssh_Archive_Downloader_When_Ssh_User_And_Archive_Download_Host_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SshUser = SSH_USER, + SshPrivateKey = SSH_PRIVATE_KEY, + SshPort = SSH_PORT, + GitlabSharedHome = BBS_SHARED_HOME, + GitlabServerUrl = BBS_SERVER_URL + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(ARCHIVE_DOWNLOAD_HOST, SSH_USER, SSH_PRIVATE_KEY, SSH_PORT, BBS_SHARED_HOME)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Smb_Archive_Downloader_Based_On_Server_Url_When_Smb_User_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + SmbUser = SMB_USER, + SmbPassword = SMB_PASSWORD, + SmbDomain = SMB_DOMAIN, + GitlabSharedHome = BBS_SHARED_HOME, + GitlabServerUrl = BBS_SERVER_URL + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(BBS_HOST, SMB_USER, SMB_PASSWORD, SMB_DOMAIN, BBS_SHARED_HOME)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Smb_Archive_Downloader_When_Smb_User_And_Archive_Download_Host_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, + SmbUser = SMB_USER, + SmbPassword = SMB_PASSWORD, + SmbDomain = SMB_DOMAIN, + GitlabSharedHome = BBS_SHARED_HOME, + GitlabServerUrl = BBS_SERVER_URL + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(ARCHIVE_DOWNLOAD_HOST, SMB_USER, SMB_PASSWORD, SMB_DOMAIN, BBS_SHARED_HOME)); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + // Arrange + var args = new MigrateRepoCommandArgs(); + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockGithubApiFactory.Verify(m => m.Create(It.IsAny(), null, It.IsAny()), Times.Never); + _mockGitlabApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockAzureApiFactory.Verify(m => m.Create(It.IsAny()), Times.Never); + _mockAzureApiFactory.Verify(m => m.CreateClientNoSsl(It.IsAny()), Times.Never); + } + + [Fact] + public void BuildHandler_Creates_GitHub_Api_When_Github_Org_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockGithubApiFactory.Verify(m => m.Create(null, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Uses_Target_Api_Url_When_Provided() + { + // Arrange + var targetApiUrl = "https://api.github.com"; + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT, + TargetApiUrl = targetApiUrl + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Api_When_Gitlab_Server_Url_Is_Provided() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, false)); + } + + [Fact] + public void BuildHandler_Creates_Azure_Api_Factory_When_Azure_Storage_Connection_String_Is_Provided_Via_Args() + { + // Arrange + var args = new MigrateRepoCommandArgs + { + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); + } + + [Fact] + public void BuildHandler_Creates_Azure_Api_Factory_When_Azure_Storage_Connection_String_Is_Provided_Via_Environment_Variables() + { + // Arrange + _mockEnvironmentVariableProvider.Setup(m => m.AzureStorageConnectionString(false)).Returns(AZURE_STORAGE_CONNECTION_STRING); + + var args = new MigrateRepoCommandArgs(); + + // Act + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + // Assert + handler.Should().NotBeNull(); + + _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); + } + + [Fact] + public void It_Gets_A_Kerberos_HttpClient_When_Kerberos_Is_True() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + Kerberos = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, false)); + } + + [Fact] + public void It_Gets_A_Kerberos_With_No_Ssl_Verify_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_True() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + Kerberos = true, + NoSslVerify = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, true)); + } + + [Fact] + public void It_Gets_A_Default_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_Not_Set() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, false)); + } + + [Fact] + public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_True() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = BBS_SERVER_URL, + GitlabUsername = BBS_USERNAME, + GitlabPassword = BBS_PASSWORD, + NoSslVerify = true + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, true)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs new file mode 100644 index 000000000..bdc05d23b --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs @@ -0,0 +1,28 @@ +using OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.ReclaimMannequin; + +public class ReclaimMannequinCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new ReclaimMannequinCommand(); + Assert.NotNull(command); + Assert.Equal("reclaim-mannequin", command.Name); + Assert.Equal(11, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "csv", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-user", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-user", false); + TestHelpers.VerifyCommandOption(command.Options, "force", false); + TestHelpers.VerifyCommandOption(command.Options, "no-prompt", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "skip-invitation", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs new file mode 100644 index 000000000..de7298f2c --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.RevokeMigratorRole; + +public class RevokeMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new RevokeMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("revoke-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs new file mode 100644 index 000000000..1c6238b62 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.WaitForMigration; + +public class WaitForMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new WaitForMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("wait-for-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs new file mode 100644 index 000000000..53309ddb5 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs @@ -0,0 +1,93 @@ +using System.Linq; +using System.Net.Http; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.bbs2gh.Factories; + +public class GitlabApiFactoryTests +{ + private const string BBS_SERVER_URL = "http://bbs.contoso.com:7990"; + + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockHttpClientFactory = new Mock(); + + private readonly GitlabApiFactory _gitlabApiFactory; + + public GitlabApiFactoryTests() + { + _gitlabApiFactory = new GitlabApiFactory(_mockOctoLogger.Object, _mockHttpClientFactory.Object, _mockEnvironmentVariableProvider.Object, null, null); + } + + [Fact] + public void Should_Create_GitlabApi_For_Source_Gitlab_Api_With_Kerberos() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("Kerberos")) + .Returns(httpClient); + + // Act + var githubApi = _gitlabApiFactory.CreateKerberos(BBS_SERVER_URL); + + // Assert + githubApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } + + [Fact] + public void Should_Create_GitlabApi_For_Source_Gitlab_Api_With_Default() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("Default")) + .Returns(httpClient); + + // Act + var githubApi = _gitlabApiFactory.Create(BBS_SERVER_URL, "user", "pass"); + + // Assert + githubApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } + + [Fact] + public void Should_Create_GitlabApi_With_No_Ssl_Verify() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("NoSSL")) + .Returns(httpClient); + + // Act + var githubApi = _gitlabApiFactory.Create(BBS_SERVER_URL, "user", "pass", true); + + // Assert + githubApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } + + [Fact] + public void Should_Create_GitlabApi_With_Kerberos_And_No_Ssl_Verify() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("KerberosNoSSL")) + .Returns(httpClient); + + // Act + var githubApi = _gitlabApiFactory.CreateKerberos(BBS_SERVER_URL, true); + + // Assert + githubApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs new file mode 100644 index 000000000..1586ff5ea --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs @@ -0,0 +1,171 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using Octoshift.Models; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands +{ + public class GitlabInspectorServiceTests + { + private readonly OctoLogger _logger = TestHelpers.CreateMock().Object; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly GitlabInspectorService _service; + + private const string BBS_FOO_PROJECT_KEY = "FP"; + private const string BBS_BAR_PROJECT_KEY = "BP"; + + public GitlabInspectorServiceTests() => _service = new(_logger, _mockGitlabApi.Object); + + [Fact] + public async Task GetProjects_Should_Return_All_Projects() + { + // Arrange + var project1 = "project1"; + var project2 = "project2"; + var projects = new[] { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: project1), + (Id: 1, Key: BBS_BAR_PROJECT_KEY, Name: project2) + }; + + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(projects); + + // Act + var result = await _service.GetProjects(); + + // Assert + result.Should().BeEquivalentTo([(BBS_FOO_PROJECT_KEY, project1), (BBS_BAR_PROJECT_KEY, project2)]); + } + + [Fact] + public async Task GetRepos_Should_Return_All_Repos() + { + // Arrange + var repo1 = "repo1"; + var repo2 = "repo2"; + var repos = new[] + { + (Id: 1, Slug: repo1, Name: repo1), + (Id: 2, Slug: repo2, Name: repo2) + }; + + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); + + // Act + var result = await _service.GetRepos(BBS_FOO_PROJECT_KEY); + + // Assert + result.Should().BeEquivalentTo(new List() { new() { Name = repo1, Slug = repo1 }, new() { Name = repo2, Slug = repo2 } }); + } + + [Fact] + public async Task GetRepoCount_Should_Return_Count() + { + // Arrange + var project = "project"; + var projects = new[] { + (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: project) + }; + var repo1 = "repo1"; + var repo2 = "repo2"; + var repos = new[] + { + (Id: 1, Slug: repo1, Name: repo1), + (Id: 2, Slug: repo2, Name: repo2) + }; + var expectedCount = 2; + + _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(projects); + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); + + // Act + var result = await _service.GetRepoCount(); + + // Assert + result.Should().Be(expectedCount); + } + + [Fact] + public async Task GetRepoCount_With_Project_Keys_Should_Return_Count() + { + // Arrange + var repo1 = "repo1"; + var repo2 = "repo2"; + var repos = new[] + { + (Id: 1, Slug: repo1, Name: repo1), + (Id: 2, Slug: repo2, Name: repo2) + }; + var expectedCount = 2; + + _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); + + // Act + var result = await _service.GetRepoCount(new[] { BBS_FOO_PROJECT_KEY }); + + // Assert + result.Should().Be(expectedCount); + _mockGitlabApi.Verify(m => m.GetProjects(), Times.Never); + } + + [Fact] + public async Task GetPullRequestCount_Should_Return_Count() + { + // Arrange + var project = "project"; + var repo1 = "repo1"; + var repo2 = "repo2"; + var repos = new[] + { + (Id: 1, Slug: repo1, Name: repo1), + (Id: 2, Slug: repo2, Name: repo2) + }; + + var prs1 = new[] + { + (Id: 1, Name: "pr1"), + (Id: 2, Name: "pr2") + }; + var prs2 = new[] + { + (Id: 3, Name: "pr3") + }; + var expectedCount = 3; + + _mockGitlabApi.Setup(m => m.GetRepos(project)).ReturnsAsync(repos); + _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo1)).ReturnsAsync(prs1); + _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo2)).ReturnsAsync(prs2); + + // Act + var result = await _service.GetPullRequestCount(project); + + // Assert + result.Should().Be(expectedCount); + } + + [Fact] + public async Task GetRepositoryPullRequestCount_Should_Return_Count() + { + // Arrange + var project = "project"; + var repo = "repo1"; + var prs = new[] + { + (Id: 1, Name: "pr1"), + (Id: 2, Name: "pr2") + }; + var expectedCount = 2; + + _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo)).ReturnsAsync(prs); + + // Act + var result = await _service.GetRepositoryPullRequestCount(project, repo); + + // Assert + result.Should().Be(expectedCount); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs new file mode 100644 index 000000000..135fd774f --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs @@ -0,0 +1,191 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using SMBLibrary; +using SMBLibrary.Client; +using Xunit; +using FileAttributes = SMBLibrary.FileAttributes; + +namespace OctoshiftCLI.Tests.bbs2gh.Services; + +public class GitlabSmbArchiveDownloaderTests +{ + private const int EXPORT_JOB_ID = 1; + private const string SHARE_ROOT = "SHARE_ROOT"; + private const string BBS_HOME_DIRECTORY_FROM_SHARE = "PATH\\TO\\BBS\\HOME\\DIRECTORY"; + private const string BBS_HOME_DIRECTORY = $"{SHARE_ROOT}\\{BBS_HOME_DIRECTORY_FROM_SHARE}"; + private const string TARGET_DIRECTORY = "TARGET"; + private const string HOST = "HOST"; + private const string SMB_USER = "SMB_USER"; + private const string SMB_PASSWORD = "SMB_PASSWORD"; + private const string DOMAIN = "DOMAIN"; + + private readonly string _exportArchiveFilename = $"Bitbucket_export_{EXPORT_JOB_ID}.tar"; + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockSmbClient = new(); + private readonly Mock _mockSmbFileStore = new(); + private readonly GitlabSmbArchiveDownloader _bbsArchiveDownloader; + + public GitlabSmbArchiveDownloaderTests() + { + _bbsArchiveDownloader = new GitlabSmbArchiveDownloader( + _mockOctoLogger.Object, + _mockFileSystemProvider.Object, + _mockSmbClient.Object, + HOST, + SMB_USER, + SMB_PASSWORD, + DOMAIN) + { GitlabSharedHomeDirectory = BBS_HOME_DIRECTORY }; + } + + [Fact] + public async Task Download_Returns_Downloaded_Archive_Full_Name() + { + // Arrange + var expectedSourceArchiveFullNameAfterShare = Path.Join(BBS_HOME_DIRECTORY_FROM_SHARE, "data/migration/export", _exportArchiveFilename).ToWindowsPath(); + var expectedTargetArchiveFullName = Path.Join(TARGET_DIRECTORY, _exportArchiveFilename).ToUnixPath(); + + _mockSmbClient.Setup(m => m.Connect(HOST, SMBTransportType.DirectTCPTransport)).Returns(true); + _mockSmbClient.Setup(m => m.Login(DOMAIN, SMB_USER, SMB_PASSWORD)).Returns(NTStatus.STATUS_SUCCESS); + var createSmbFileStoreStatus = NTStatus.STATUS_SUCCESS; + _mockSmbClient.Setup(m => m.TreeConnect(SHARE_ROOT, out createSmbFileStoreStatus)).Returns(_mockSmbFileStore.Object); + + var sharedFileHandle = new object(); + var fileStatus = FileStatus.FILE_OPENED; + _mockSmbFileStore.Setup(m => m.CreateFile( + out sharedFileHandle, + out fileStatus, + expectedSourceArchiveFullNameAfterShare, + AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.Read, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null)) + .Returns(NTStatus.STATUS_SUCCESS); + + FileInformation fileStandardInformation = new FileStandardInformation + { + AllocationSize = 10 * 1024 * 1024 // 10 MB + }; + _mockSmbFileStore + .Setup(m => m.GetFileInformation(out fileStandardInformation, sharedFileHandle, FileInformationClass.FileStandardInformation)) + .Returns(NTStatus.STATUS_SUCCESS); + + var data = new byte[1024]; + _mockSmbFileStore + .SetupSequence(m => m.ReadFile(out data, sharedFileHandle, It.IsAny(), It.IsAny())) + .Returns(NTStatus.STATUS_SUCCESS) + .Returns(NTStatus.STATUS_SUCCESS) + .Returns(NTStatus.STATUS_END_OF_FILE); + + // Act + var actualTargetArchiveFullName = await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); + + // Assert + _mockSmbClient.Verify(m => m.Connect(HOST, SMBTransportType.DirectTCPTransport), Times.Once); + _mockSmbClient.Verify(m => m.Login(DOMAIN, SMB_USER, SMB_PASSWORD), Times.Once); + _mockSmbClient.Verify(m => m.TreeConnect(SHARE_ROOT, out createSmbFileStoreStatus), Times.Once); + _mockSmbFileStore.Verify(m => m.CreateFile( + out sharedFileHandle, + out fileStatus, + expectedSourceArchiveFullNameAfterShare, + AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.Read, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null), + Times.Once); + _mockSmbFileStore.Verify(m => m.GetFileInformation(out fileStandardInformation, sharedFileHandle, FileInformationClass.FileStandardInformation), Times.Once); + _mockSmbFileStore.Verify(m => m.ReadFile(out data, sharedFileHandle, It.IsAny(), It.IsAny()), Times.Exactly(3)); + _mockFileSystemProvider.Verify(m => m.CreateDirectory(TARGET_DIRECTORY), Times.Once); + _mockFileSystemProvider.Verify(m => m.Open(expectedTargetArchiveFullName, FileMode.Create), Times.Once); + _mockFileSystemProvider.Verify(m => m.WriteAsync(It.IsAny(), data, It.IsAny()), Times.Exactly(2)); + + actualTargetArchiveFullName.Should().Be(expectedTargetArchiveFullName); + } + + [Fact] + public async Task Download_Throws_When_Cannot_Connect_To_Host() + { + // Arrange + _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(false); + + // Act, Assert + await _bbsArchiveDownloader + .Invoking(async x => await x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) + .Should() + .ThrowExactlyAsync(); + } + + [Fact] + public async Task Download_Throws_When_Cannot_Login() + { + // Arrange + _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); + _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_LOGON_FAILURE); + + // Act, Assert + await _bbsArchiveDownloader + .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"*{NTStatus.STATUS_LOGON_FAILURE}*"); + } + + [Fact] + public async Task Download_Throws_When_Cannot_Connect_To_Share() + { + // Arrange + _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); + _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_SUCCESS); + var status = NTStatus.STATUS_BAD_NETWORK_NAME; + _mockSmbClient.Setup(m => m.TreeConnect(It.IsAny(), out status)).Returns(_mockSmbFileStore.Object); + + // Act, Assert + await _bbsArchiveDownloader + .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"*{NTStatus.STATUS_BAD_NETWORK_NAME}*"); + } + + [Fact] + public async Task Download_Throws_When_Source_Export_Archive_Does_Not_Exist() + { + // Arrange + _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); + _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_SUCCESS); + var status = NTStatus.STATUS_SUCCESS; + _mockSmbClient.Setup(m => m.TreeConnect(It.IsAny(), out status)).Returns(_mockSmbFileStore.Object); + + object sharedFileHandle; + FileStatus fileStatus; + _mockSmbFileStore.Setup(m => m.CreateFile( + out sharedFileHandle, + out fileStatus, + It.IsAny(), + AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.Read, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null)) + .Returns(NTStatus.STATUS_OBJECT_NAME_NOT_FOUND); + + // Act, Assert + await _bbsArchiveDownloader + .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"*{NTStatus.STATUS_OBJECT_NAME_NOT_FOUND}*"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs new file mode 100644 index 000000000..94e407df5 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs @@ -0,0 +1,85 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Services; +using Renci.SshNet; +using Xunit; + +namespace OctoshiftCLI.Tests.bbs2gh.Services; + +public sealed class GitlabSshArchiveDownloaderTests : IDisposable +{ + private const int EXPORT_JOB_ID = 1; + private const string BBS_HOME_DIRECTORY = "BBS_HOME"; + private const string TARGET_DIRECTORY = "TARGET"; + + private readonly string _exportArchiveFilename = $"Bitbucket_export_{EXPORT_JOB_ID}.tar"; + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockSftpClient = new(); + private readonly GitlabSshArchiveDownloader _bbsArchiveDownloader; + + public GitlabSshArchiveDownloaderTests() + { + _bbsArchiveDownloader = new GitlabSshArchiveDownloader(_mockOctoLogger.Object, _mockFileSystemProvider.Object, _mockSftpClient.Object) + { + GitlabSharedHomeDirectory = BBS_HOME_DIRECTORY + }; + + _mockSftpClient.Setup(m => m.Exists(It.IsAny())).Returns(true); + + var mockAsyncResult = new Mock(); + mockAsyncResult.Setup(m => m.IsCompleted).Returns(true); + _mockSftpClient + .Setup(m => m.BeginDownloadFile(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .Returns(mockAsyncResult.Object); + } + + [Fact] + public async Task Download_Returns_Downloaded_Archive_Full_Name() + { + // Arrange + var expectedSourceArchiveFullName = Path.Join(BBS_HOME_DIRECTORY, "data/migration/export", _exportArchiveFilename).Replace('\\', '/'); + var expectedTargetArchiveFullName = Path.Join(TARGET_DIRECTORY, _exportArchiveFilename).Replace('\\', '/'); + + // Act + var actualDownloadedArchiveFullName = await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); + + // Assert + _mockSftpClient.Verify(m => + m.BeginDownloadFile( + expectedSourceArchiveFullName, + It.IsAny(), + null, + null, + It.IsAny>())); + + _mockFileSystemProvider.Verify(m => m.Open(expectedTargetArchiveFullName, FileMode.Create)); + actualDownloadedArchiveFullName.Should().Be(expectedTargetArchiveFullName); + } + + [Fact] + public async Task Download_Throws_When_Source_Export_Archive_Does_Not_Exist() + { + // Arrange + _mockSftpClient.Setup(m => m.Exists(It.IsAny())).Returns(false); + + // Act, Assert + await _bbsArchiveDownloader.Invoking(x => x.Download(EXPORT_JOB_ID)).Should().ThrowExactlyAsync(); + } + + [Fact] + public async Task Download_Creates_Target_Directory() + { + // Arrange, Act + await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); + + // Assert + _mockFileSystemProvider.Verify(m => m.CreateDirectory(TARGET_DIRECTORY), Times.Once); + } + + public void Dispose() => _bbsArchiveDownloader?.Dispose(); +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs new file mode 100644 index 000000000..8086eb131 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands +{ + public class ProjectsCsvGeneratorServiceTests + { + private const string FULL_CSV_HEADER = "project-key,project-name,url,repo-count,pr-count"; + private const string MINIMAL_CSV_HEADER = "project-key,project-name,url,repo-count"; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + + private const string BBS_SERVER_URL = "http://bbs-server-url"; + private const string BBS_FOO_PROJECT = "project1"; + private const string BBS_BAR_PROJECT = "project2"; + private const string BBS_FOO_PROJECT_KEY = "FP"; + private const string BBS_BAR_PROJECT_KEY = "BP"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const bool NO_SSL_VERIFY = true; + private readonly (string, string) _bbsProject = (BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT); + private readonly IEnumerable<(string, string)> _bbsProjects = [(BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT), (BBS_BAR_PROJECT_KEY, BBS_BAR_PROJECT)]; + + private readonly ProjectsCsvGeneratorService _service; + + public ProjectsCsvGeneratorServiceTests() + { + _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); + _service = new ProjectsCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); + } + + [Fact] + public async Task Generate_Should_Return_Correct_Csv_For_One_Project() + { + // Arrange + var repoCount = 82; + var prCount = 822; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repoCount); + _mockGitlabInspectorService.Setup(m => m.GetPullRequestCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(prCount); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); + + // Assert + var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; + + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}\",{repoCount},{prCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Should_Return_Minimal_Csv_When_Minimal_Is_True() + { + // Arrange + const int repoCount1 = 82; + const int repoCount2 = 0; + const bool minimal = true; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProjects()).ReturnsAsync(_bbsProjects); + _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repoCount1); + _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_BAR_PROJECT_KEY)).ReturnsAsync(repoCount2); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, "", minimal); + + // Assert + var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}\",{repoCount1}{Environment.NewLine}"; + expected += $"\"{BBS_BAR_PROJECT_KEY}\",\"{BBS_BAR_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_BAR_PROJECT_KEY}\",{repoCount2}{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs new file mode 100644 index 000000000..87af627f6 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using Octoshift.Models; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands +{ + public class ReposCsvGeneratorServiceTests + { + private const string FULL_CSV_HEADER = "project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived,pr-count"; + private const string MINIMAL_CSV_HEADER = "project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + + private const string BBS_SERVER_URL = "http://bbs-server-url"; + private const string BBS_FOO_PROJECT = "project"; + private const string BBS_FOO_PROJECT_KEY = "FP"; + private const string BBS_USERNAME = "bbs-username"; + private const string BBS_PASSWORD = "bbs-password"; + private const bool NO_SSL_VERIFY = true; + private readonly (string, string) _bbsProject = (BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT); + private const string BBS_REPO = "foo-repo"; + private const string BBS_REPO_SLUG = "foo-repo-slug"; + private const bool ARCHIVED = false; + private const ulong REPO_SIZE = 10000UL; + private const ulong ATTACHMENTS_SIZE = 10000UL; + private readonly IEnumerable _bbsRepos = [new() { Name = BBS_REPO, Slug = BBS_REPO_SLUG }]; + + private readonly ReposCsvGeneratorService _service; + + public ReposCsvGeneratorServiceTests() + { + _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); + _service = new ReposCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); + } + + [Fact] + public async Task Generate_Should_Return_Correct_Csv_For_One_Repo() + { + // Arrange + var prCount = 822; + var lastCommitDate = DateTime.Now; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); + _mockGitlabInspectorService.Setup(m => m.GetRepositoryPullRequestCount(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(prCount); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetIsRepositoryArchived(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(ARCHIVED); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); + + // Assert + var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\",\"False\",{prCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Should_Return_Correct_Csv_For_One_Repo_Without_Archived_Field_For_Outdated_BBS_Version() + { + // Arrange + var prCount = 822; + var lastCommitDate = DateTime.Now; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); + _mockGitlabInspectorService.Setup(m => m.GetRepositoryPullRequestCount(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(prCount); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetIsRepositoryArchived(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(ARCHIVED); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); + + // Assert + var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\",\"False\",{prCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Should_Return_Minimal_Csv_When_Minimal_Is_True() + { + // Arrange + var lastCommitDate = DateTime.Now; + const bool minimal = true; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); + + // Assert + var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Generate_Should_Include_Empty_Entry_For_Null_Latest_Commit_Date() + { + // Arrange + const bool minimal = true; + + var project_name = "project,name"; + var repo_name = "repo,name"; + var expected_project_name = "project%2Cname"; + var expected_repo_name = "repo%2Cname"; + var bbsProject = (BBS_FOO_PROJECT_KEY, project_name); + var bbsRepos = new List { new() { Name = repo_name, Slug = BBS_REPO_SLUG } }; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsRepos); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); + + // Assert + var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{expected_project_name}\",\"{expected_repo_name}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",,\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Generate_Should_Escape_Project_And_Repo_Names() + { + // Arrange + var lastCommitDate = DateTime.Now; + const bool minimal = true; + + var project_name = "project,name"; + var repo_name = "repo,name"; + var expected_project_name = "project%2Cname"; + var expected_repo_name = "repo%2Cname"; + var bbsProject = (BBS_FOO_PROJECT_KEY, project_name); + var bbsRepos = new List { new() { Name = repo_name, Slug = BBS_REPO_SLUG } }; + + _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + + _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsProject); + _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsRepos); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); + + // Act + var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); + + // Assert + var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; + expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{expected_project_name}\",\"{expected_repo_name}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); + } + } +} diff --git a/src/OctoshiftCLI.sln b/src/OctoshiftCLI.sln index 72aee8119..17c0dcb4b 100644 --- a/src/OctoshiftCLI.sln +++ b/src/OctoshiftCLI.sln @@ -21,6 +21,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OctoshiftCLI.IntegrationTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "bbs2gh", "bbs2gh\bbs2gh.csproj", "{39EE734C-AC4A-42AC-801A-ECCA661C23A1}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gl2gh", "gl2gh\gl2gh.csproj", "{180D1E3B-CA23-4603-BE00-9C856E0665CF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +53,10 @@ Global {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Debug|Any CPU.Build.0 = Debug|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.ActiveCfg = Release|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.Build.0 = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs new file mode 100644 index 000000000..06fc255ec --- /dev/null +++ b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.AbortMigration; + +namespace OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; + +public sealed class AbortMigrationCommand : AbortMigrationCommandBase +{ + public AbortMigrationCommand() : base() => AddOptions(); +} diff --git a/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs new file mode 100644 index 000000000..56911ff57 --- /dev/null +++ b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.CreateTeam; + +namespace OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; + +public sealed class CreateTeamCommand : CreateTeamCommandBase +{ + public CreateTeamCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs new file mode 100644 index 000000000..52adb4ce4 --- /dev/null +++ b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs @@ -0,0 +1,17 @@ +using System.Runtime.CompilerServices; +using OctoshiftCLI.Commands.DownloadLogs; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] + +namespace OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; + +public sealed class DownloadLogsCommand : DownloadLogsCommandBase +{ + public DownloadLogsCommand() + { + // Add backward compatibility alias for --github-api-url + GithubApiUrl.AddAlias("--github-api-url"); + + AddOptions(); + } +} diff --git a/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs new file mode 100644 index 000000000..7632ec845 --- /dev/null +++ b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GenerateMannequinCsv; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; + +public sealed class GenerateMannequinCsvCommand : GenerateMannequinCsvCommandBase +{ + public GenerateMannequinCsvCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs new file mode 100644 index 000000000..1210a5058 --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -0,0 +1,159 @@ +using System; +using System.CommandLine; +using System.IO; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommand : CommandBase +{ + public GenerateScriptCommand() : base( + name: "generate-script", + description: "Generates a migration script. This provides you the ability to review the steps that this tool will take, and optionally modify the script if desired before running it.") + { + AddOption(GitlabServerUrl); + AddOption(GithubOrg); + AddOption(TargetApiUrl); + AddOption(GitlabUsername); + AddOption(GitlabPassword); + AddOption(GitlabProject); + AddOption(GitlabSharedHome); + AddOption(SshUser); + AddOption(SshPrivateKey); + AddOption(SshPort); + AddOption(ArchiveDownloadHost); + AddOption(SmbUser); + AddOption(SmbDomain); + AddOption(Output); + AddOption(Kerberos); + AddOption(Verbose); + AddOption(AwsBucketName); + AddOption(AwsRegion); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--bbs-server-url", + description: "The full URL of the Bitbucket Server/Data Center to migrate from.") + { IsRequired = true }; + + public Option GitlabUsername { get; } = new( + name: "--bbs-username", + description: "The Bitbucket username of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_USERNAME environment variable."); + + public Option GitlabPassword { get; } = new( + name: "--bbs-password", + description: "The Bitbucket password of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_PASSWORD environment variable." + + $"{Environment.NewLine}" + + "Note: The password will not get included in the generated script and it has to be set as an env variable before running the script."); + + public Option GitlabProject { get; } = new( + name: "--bbs-project", + description: "The Bitbucket project to migrate. If not set will migrate all projects."); + + public Option GitlabSharedHome { get; } = new( + name: "--bbs-shared-home", + description: "Bitbucket server's shared home directory. Defaults to \"/var/atlassian/application-data/bitbucket/shared\" if downloading the archive from a server using SSH " + + "and \"c$\\atlassian\\applicationdata\\bitbucket\\shared\" if downloading using SMB."); + + public Option ArchiveDownloadHost { get; } = new( + name: "--archive-download-host", + description: "The host to use to connect to the Bitbucket Server/Data Center instance via SSH or SMB. Defaults to the host from the Bitbucket Server URL (--bbs-server-url)."); + + public Option SshUser { get; } = new( + name: "--ssh-user", + description: "The SSH user to be used for downloading the export archive off of the Bitbucket server."); + + public Option SshPrivateKey { get; } = new( + name: "--ssh-private-key", + description: "The full path of the private key file to be used for downloading the export archive off of the Bitbucket Server using SSH/SFTP."); + + public Option SshPort { get; } = new( + name: "--ssh-port", + description: "The SSH port (default: 22).", + getDefaultValue: () => 22); + + public Option SmbUser { get; } = new( + name: "--smb-user", + description: "The SMB user used for authentication when downloading the export archive from the Bitbucket Server instance." + + $"{Environment.NewLine}" + + "Note: You must also specify the SMB password using the SMB_PASSWORD environment variable."); + + public Option SmbDomain { get; } = new( + name: "--smb-domain", + description: "The optional domain name when using SMB for downloading the export archive."); + + public Option GithubOrg { get; } = new("--github-org") + { IsRequired = true }; + + public Option Output { get; } = new( + name: "--output", + getDefaultValue: () => new FileInfo("./migrate.ps1")); + + public Option Kerberos { get; } = new( + name: "--kerberos", + description: "Use Kerberos authentication for Bitbucket Server.") + { IsHidden = true }; + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the BBS archive to."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. All other migration steps will continue to verify SSL. " + + "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow the migration archive to be exported."); + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).", + }; + + public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var versionProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + var environmentVariableProvider = sp.GetRequiredService(); + + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = args.Kerberos + ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + + return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi, environmentVariableProvider); + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs new file mode 100644 index 000000000..709fd55de --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -0,0 +1,55 @@ +using System.IO; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgs : CommandArgs +{ + public string GitlabServerUrl { get; set; } + public string GithubOrg { get; set; } + public string GitlabUsername { get; set; } + [Secret] + public string GitlabPassword { get; set; } + public string GitlabProject { get; set; } + public string GitlabSharedHome { get; set; } + public string ArchiveDownloadHost { get; set; } + public string SshUser { get; set; } + public string SshPrivateKey { get; set; } + public int SshPort { get; set; } + public string SmbUser { get; set; } + public string SmbDomain { get; set; } + public FileInfo Output { get; set; } + public bool Kerberos { get; set; } + public string AwsBucketName { get; set; } + public string AwsRegion { get; set; } + public bool KeepArchive { get; set; } + public bool NoSslVerify { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (NoSslVerify && GitlabServerUrl.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); + } + + if (AwsBucketName.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + if (AwsRegion.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } + + if (SshPort == 7999) + { + log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); + } + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs new file mode 100644 index 000000000..b7878ea23 --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -0,0 +1,213 @@ +using System; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly IVersionProvider _versionProvider; + private readonly FileSystemProvider _fileSystemProvider; + private readonly GitlabApi _gitlabApi; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + + public GenerateScriptCommandHandler( + OctoLogger log, + IVersionProvider versionProvider, + FileSystemProvider fileSystemProvider, + GitlabApi gitlabApi, + EnvironmentVariableProvider environmentVariableProvider) + { + _log = log; + _versionProvider = versionProvider; + _fileSystemProvider = fileSystemProvider; + _gitlabApi = gitlabApi; + _environmentVariableProvider = environmentVariableProvider; + } + + public async Task Handle(GenerateScriptCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Generating Script..."); + + var script = await GenerateScript(args); + + if (script.HasValue() && args.Output.HasValue()) + { + await _fileSystemProvider.WriteAllTextAsync(args.Output.FullName, script); + } + } + + private async Task GenerateScript(GenerateScriptCommandArgs args) + { + var content = new StringBuilder(); + content.AppendLine(PWSH_SHEBANG); + content.AppendLine(); + content.AppendLine(VersionComment); + content.AppendLine(EXEC_FUNCTION_BLOCK); + + content.AppendLine(VALIDATE_GH_PAT); + if (!args.Kerberos) + { + content.AppendLine(VALIDATE_BBS_PASSWORD); + } + if (args.GitlabUsername.IsNullOrWhiteSpace() && !args.Kerberos) + { + content.AppendLine(VALIDATE_BBS_USERNAME); + } + if (args.AwsBucketName.HasValue() || args.AwsRegion.HasValue()) + { + content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); + content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY); + } + else if (!args.UseGithubStorage) + { + content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); + } + if (args.SmbUser.HasValue()) + { + content.AppendLine(VALIDATE_SMB_PASSWORD); + } + + var projects = args.GitlabProject.HasValue() + ? [args.GitlabProject] + : (await _gitlabApi.GetProjects()).Select(x => x.Key); + + foreach (var projectKey in projects) + { + _log.LogInformation($"Project: {projectKey}"); + + content.AppendLine(); + content.AppendLine($"# =========== Project: {projectKey} ==========="); + + var repos = await _gitlabApi.GetRepos(projectKey); + + if (!repos.Any()) + { + content.AppendLine("# Skipping this project because it has no git repos."); + continue; + } + + content.AppendLine(); + + foreach (var (_, repoSlug, repoName) in repos) + { + _log.LogInformation($" Repo: {repoName}"); + + content.AppendLine(Exec(MigrateGithubRepoScript(args, projectKey, repoSlug, true))); + } + } + + return content.ToString(); + } + + private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bbsProjectKey, string bbsRepoSlug, bool wait) + { + var bbsServerUrlOption = $" --bbs-server-url \"{args.GitlabServerUrl}\""; + var bbsUsernameOption = args.GitlabUsername.HasValue() ? $" --bbs-username \"{args.GitlabUsername}\"" : ""; + var bbsProjectOption = $" --bbs-project \"{bbsProjectKey}\""; + var bbsRepoOption = $" --bbs-repo \"{bbsRepoSlug}\""; + var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; + var githubRepoOption = $" --github-repo \"{GetGithubRepoName(bbsProjectKey, bbsRepoSlug)}\""; + var waitOption = wait ? "" : " --queue-only"; + var kerberosOption = args.Kerberos ? " --kerberos" : ""; + var verboseOption = args.Verbose ? " --verbose" : ""; + var sshArchiveDownloadOptions = args.SshUser.HasValue() + ? $" --ssh-user \"{args.SshUser}\" --ssh-private-key \"{args.SshPrivateKey}\"{(args.SshPort.HasValue() ? $" --ssh-port {args.SshPort}" : "")}{(args.ArchiveDownloadHost.HasValue() ? $" --archive-download-host {args.ArchiveDownloadHost}" : "")}" : ""; + var smbArchiveDownloadOptions = args.SmbUser.HasValue() + ? $" --smb-user \"{args.SmbUser}\"{(args.SmbDomain.HasValue() ? $" --smb-domain {args.SmbDomain}" : "")}{(args.ArchiveDownloadHost.HasValue() ? $" --archive-download-host {args.ArchiveDownloadHost}" : "")}" + : ""; + var bbsSharedHomeOption = args.GitlabSharedHome.HasValue() ? $" --bbs-shared-home \"{args.GitlabSharedHome}\"" : ""; + var awsBucketNameOption = args.AwsBucketName.HasValue() ? $" --aws-bucket-name \"{args.AwsBucketName}\"" : ""; + var awsRegionOption = args.AwsRegion.HasValue() ? $" --aws-region \"{args.AwsRegion}\"" : ""; + var keepArchive = args.KeepArchive ? " --keep-archive" : ""; + var noSslVerify = args.NoSslVerify ? " --no-ssl-verify" : ""; + var targetRepoVisibility = " --target-repo-visibility private"; + var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; + var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; + var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; + + return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsSharedHomeOption}{bbsProjectOption}{bbsRepoOption}{sshArchiveDownloadOptions}" + + $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; + } + + private string Exec(string script) => Wrap(script, "Exec"); + + private string Wrap(string script, string outerCommand = "") => script.IsNullOrWhiteSpace() ? string.Empty : $"{outerCommand} {{ {script} }}".Trim(); + + private string GetGithubRepoName(string bbsProjectKey, string bbsRepoSlug) => $"{bbsProjectKey}-{bbsRepoSlug}".ReplaceInvalidCharactersWithDash(); + + private string VersionComment => $"# =========== Created with CLI version {_versionProvider.GetCurrentVersion()} ==========="; + + private const string PWSH_SHEBANG = "#!/usr/bin/env pwsh"; + + private const string EXEC_FUNCTION_BLOCK = @" +function Exec { + param ( + [scriptblock]$ScriptBlock + ) + & @ScriptBlock + if ($lastexitcode -ne 0) { + exit $lastexitcode + } +}"; + private const string VALIDATE_GH_PAT = @" +if (-not $env:GH_PAT) { + Write-Error ""GH_PAT environment variable must be set to a valid GitHub Personal Access Token with the appropriate scopes. For more information see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#creating-a-personal-access-token-for-github-enterprise-importer"" + exit 1 +} else { + Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" +}"; + private const string VALIDATE_BBS_USERNAME = @" +if (-not $env:BBS_USERNAME) { + Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" +}"; + private const string VALIDATE_BBS_PASSWORD = @" +if (-not $env:BBS_PASSWORD) { + Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + exit 1 +} else { + Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" +}"; + private const string VALIDATE_AZURE_STORAGE_CONNECTION_STRING = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + private const string VALIDATE_AWS_ACCESS_KEY_ID = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + private const string VALIDATE_AWS_SECRET_ACCESS_KEY = @" +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + private const string VALIDATE_SMB_PASSWORD = @" +if (-not $env:SMB_PASSWORD) { + Write-Error ""SMB_PASSWORD environment variable must be set to a valid password that will be used to download the migration archive from your BBS server using SMB."" + exit 1 +} else { + Write-Host ""SMB_PASSWORD environment variable is set and will be used to download the migration archive from your BBS server using SMB."" +}"; +} diff --git a/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs new file mode 100644 index 000000000..fdd3ec62e --- /dev/null +++ b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GrantMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; + +public sealed class GrantMigratorRoleCommand : GrantMigratorRoleCommandBase +{ + public GrantMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs new file mode 100644 index 000000000..5aca971a6 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -0,0 +1,83 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommand : CommandBase + { + public InventoryReportCommand() : base( + name: "inventory-report", + description: "Generates several CSV files containing lists of BBS projects and repos. Useful for planning large migrations. Personal repositories owned by individual users will not be included." + + Environment.NewLine + + "Note: Expects BBS_USERNAME and BBS_PASSWORD env variables or --bbs-username and --bbs-password options to be set.") + { + AddOption(GitlabServerUrl); + AddOption(GitlabProject); + AddOption(GitlabUsername); + AddOption(GitlabPassword); + AddOption(NoSslVerify); + AddOption(Minimal); + AddOption(Verbose); + } + + public Option GitlabServerUrl { get; } = new( + name: "--bbs-server-url", + description: "The full URL of the Bitbucket Server/Data Center. E.g. http://bitbucket.contoso.com:7990") + { IsRequired = true }; + + public Option GitlabProject { get; } = new( + name: "--bbs-project", + description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); + + public Option GitlabUsername { get; } = new( + name: "--bbs-username", + description: "The Bitbucket username of a user with site admin privileges. If not set will be read from BBS_USERNAME environment variable."); + + public Option GitlabPassword { get; } = new( + name: "--bbs-password", + description: "The Bitbucket password of the user specified by --bbs-username. If not set will be read from BBS_PASSWORD environment variable."); + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. " + + "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow data to be extracted."); + + public Option Minimal { get; } = new( + name: "--minimal", + description: "Significantly speeds up the generation of the CSV files by including the bare minimum info. Will omit the archived state and PR count for repos and the PR count for projects."); + + public Option Verbose { get; } = new("--verbose"); + + public override InventoryReportCommandHandler BuildHandler(InventoryReportCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + var bbsInspectorServiceFactory = sp.GetRequiredService(); + var bbsInspectorService = bbsInspectorServiceFactory.Create(gitlabApi); + var projectsCsvGeneratorService = sp.GetRequiredService(); + var reposCsvGeneratorService = sp.GetRequiredService(); + + return new InventoryReportCommandHandler( + log, + gitlabApi, + bbsInspectorService, + projectsCsvGeneratorService, + reposCsvGeneratorService); + } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs new file mode 100644 index 000000000..3b8565f2d --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -0,0 +1,15 @@ +using OctoshiftCLI.Commands; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandArgs : CommandArgs + { + public string GitlabServerUrl { get; set; } + public string GitlabProject { get; set; } + public string GitlabUsername { get; set; } + [Secret] + public string GitlabPassword { get; set; } + public bool NoSslVerify { get; set; } + public bool Minimal { get; set; } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs new file mode 100644 index 000000000..19a3783fc --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -0,0 +1,66 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandler : ICommandHandler +{ + internal Func WriteToFile = async (path, contents) => await File.WriteAllTextAsync(path, contents); + + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + private readonly GitlabInspectorService _bbsInspectorService; + private readonly ProjectsCsvGeneratorService _projectsCsvGenerator; + private readonly ReposCsvGeneratorService _reposCsvGenerator; + + public InventoryReportCommandHandler( + OctoLogger log, + GitlabApi gitlabApi, + GitlabInspectorService bbsInspectorService, + ProjectsCsvGeneratorService projectsCsvGeneratorService, + ReposCsvGeneratorService reposCsvGeneratorService) + { + _log = log; + _gitlabApi = gitlabApi; + _bbsInspectorService = bbsInspectorService; + _projectsCsvGenerator = projectsCsvGeneratorService; + _reposCsvGenerator = reposCsvGeneratorService; + } + + public async Task Handle(InventoryReportCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Creating inventory report..."); + + var projectKeys = Array.Empty(); + if (string.IsNullOrWhiteSpace(args.GitlabProject)) + { + _log.LogInformation("Finding Projects..."); + var projects = await _gitlabApi.GetProjects(); + projectKeys = projects.Select(x => x.Key).ToArray(); + _log.LogInformation($"Found {projects.Count()} Projects"); + } + + _log.LogInformation("Finding Repos..."); + var repoCount = string.IsNullOrWhiteSpace(args.GitlabProject) ? await _bbsInspectorService.GetRepoCount(projectKeys) : await _bbsInspectorService.GetRepoCount(args.GitlabProject); + _log.LogInformation($"Found {repoCount} Repos"); + + _log.LogInformation("Generating data for projects.csv..."); + var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabProject, args.Minimal); + await WriteToFile("projects.csv", projectsCsvText); + _log.LogSuccess("projects.csv generated"); + + _log.LogInformation("Generating repos.csv..."); + var reposCsvText = await _reposCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabProject, args.Minimal); + await WriteToFile("repos.csv", reposCsvText); + _log.LogSuccess("repos.csv generated"); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs new file mode 100644 index 000000000..78d97f905 --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -0,0 +1,271 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommand : CommandBase +{ + public MigrateRepoCommand() : base( + name: "migrate-repo", + description: "Import a Bitbucket Server archive to GitHub." + + Environment.NewLine + + "Note: Expects GH_PAT env variable or --github-pat option to be set.") + { + AddOption(ArchiveUrl); + AddOption(GithubOrg); + AddOption(GithubRepo); + AddOption(GithubPat); + AddOption(GitlabServerUrl); + AddOption(GitlabProject); + AddOption(GitlabRepo); + AddOption(GitlabUsername); + AddOption(GitlabPassword); + AddOption(GitlabSharedHome); + AddOption(SshUser); + AddOption(SshPrivateKey); + AddOption(SshPort); + AddOption(ArchiveDownloadHost); + AddOption(SmbUser); + AddOption(SmbPassword); + AddOption(SmbDomain); + AddOption(ArchivePath); + AddOption(AzureStorageConnectionString); + AddOption(AwsBucketName); + AddOption(AwsAccessKey); + AddOption(AwsSecretKey); + AddOption(AwsSessionToken); + AddOption(AwsRegion); + AddOption(QueueOnly); + AddOption(TargetRepoVisibility.FromAmong("public", "private", "internal")); + AddOption(Kerberos); + AddOption(Verbose); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--bbs-server-url", + description: "The full URL of the Bitbucket Server/Data Center to migrate from. E.g. http://bitbucket.contoso.com:7990") + { + IsRequired = true + }; + + public Option GitlabProject { get; } = new( + name: "--bbs-project", + description: "The Bitbucket project to migrate.") + { + IsRequired = true + }; + + public Option GitlabRepo { get; } = new( + name: "--bbs-repo", + description: "The Bitbucket repository to migrate.") + { + IsRequired = true + }; + + public Option GitlabUsername { get; } = new( + name: "--bbs-username", + description: "The Bitbucket username of a user with site admin privileges. If not set will be read from BBS_USERNAME environment variable."); + + public Option GitlabPassword { get; } = new( + name: "--bbs-password", + description: "The Bitbucket password of the user specified by --bbs-username. If not set will be read from BBS_PASSWORD environment variable."); + + public Option GitlabSharedHome { get; } = new( + name: "--bbs-shared-home", + description: "Bitbucket server's shared home directory. Defaults to \"/var/atlassian/application-data/bitbucket/shared\" if downloading the archive from a server using SSH " + + "and \"c$\\atlassian\\applicationdata\\bitbucket\\shared\" if downloading using SMB."); + + public Option ArchiveUrl { get; } = new( + name: "--archive-url", + description: + "URL used to download Bitbucket Server migration archive. Only needed if you want to manually retrieve the archive from BBS instead of letting this CLI do that for you."); + + public Option ArchivePath { get; } = new( + name: "--archive-path", + description: "Path to Bitbucket Server migration archive on disk."); + + public Option AzureStorageConnectionString { get; } = new( + name: "--azure-storage-connection-string", + description: "A connection string for an Azure Storage account, used to upload the BBS archive. If not set will be read from AZURE_STORAGE_CONNECTION_STRING environment variable."); + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the BBS archive to."); + + public Option AwsAccessKey { get; } = new( + name: "--aws-access-key", + description: "If uploading to S3, the AWS access key. If not provided, it will be read from AWS_ACCESS_KEY_ID environment variable."); + + public Option AwsSecretKey { get; } = new( + name: "--aws-secret-key", + description: "If uploading to S3, the AWS secret key. If not provided, it will be read from AWS_SECRET_ACCESS_KEY environment variable."); + + public Option AwsSessionToken { get; } = new( + name: "--aws-session-token", + description: "If using AWS, the AWS session token. If not provided, it will be read from AWS_SESSION_TOKEN environment variable."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option GithubOrg { get; } = new("--github-org"); + + public Option GithubRepo { get; } = new("--github-repo"); + + public Option ArchiveDownloadHost { get; } = new( + name: "--archive-download-host", + description: "The host to use to connect to the Bitbucket Server/Data Center instance via SSH or SMB. Defaults to the host from the Bitbucket Server URL (--bbs-server-url)."); + + public Option SshUser { get; } = new( + name: "--ssh-user", + description: "The SSH user to be used for downloading the export archive off of the Bitbucket server."); + + public Option SshPrivateKey { get; } = new( + name: "--ssh-private-key", + description: "The full path of the private key file to be used for downloading the export archive off of the Bitbucket Server using SSH/SFTP." + + Environment.NewLine + + "Supported private key formats:" + + Environment.NewLine + + " - RSA in OpenSSL PEM format." + + Environment.NewLine + + " - DSA in OpenSSL PEM format." + + Environment.NewLine + + " - ECDSA 256/384/521 in OpenSSL PEM format." + + Environment.NewLine + + " - ECDSA 256/384/521, ED25519 and RSA in OpenSSH key format."); + + public Option SshPort { get; } = new( + name: "--ssh-port", + getDefaultValue: () => 22, + description: "The SSH port (default: 22)."); + + public Option SmbUser { get; } = new( + name: "--smb-user", + description: "The SMB user used for authentication when downloading the export archive from the Bitbucket Server instance."); + + public Option SmbPassword { get; } = new( + name: "--smb-password", + description: "The SMB password used for authentication when downloading the export archive from the Bitbucket server instance. If not provided, it will be read from SMB_PASSWORD environment variable."); + + public Option SmbDomain { get; } = new( + name: "--smb-domain", + description: "The optional domain name when using SMB for downloading the export archive."); + + public Option GithubPat { get; } = new( + name: "--github-pat", + description: "The GitHub personal access token to be used for the migration. If not set will be read from GH_PAT environment variable."); + + public Option QueueOnly { get; } = new( + name: "--queue-only", + description: "Only queues the migration, does not wait for it to finish. Use the wait-for-migration command to subsequently wait for it to finish and view the status."); + + public Option TargetRepoVisibility { get; } = new( + name: "--target-repo-visibility", + description: "The visibility of the target repo. Defaults to private. Valid values are public, private, or internal."); + + public Option Kerberos { get; } = new( + name: "--kerberos", + description: "Use Kerberos authentication for downloading the export archive off of the Bitbucket server.") + { IsHidden = true }; + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. All other migration steps will continue to verify SSL. " + + "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow the migration archive to be exported."); + public Option UseGithubStorage { get; } = new( + name: "--use-github-storage", + description: "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).") + { IsHidden = true }; + + public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var environmentVariableProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + GithubApi githubApi = null; + GitlabApi gitlabApi = null; + IGitlabArchiveDownloader bbsArchiveDownloader = null; + AzureApi azureApi = null; + AwsApi awsApi = null; + + if (args.GithubOrg.HasValue()) + { + var githubApiFactory = sp.GetRequiredService(); + githubApi = githubApiFactory.Create(args.TargetApiUrl, args.TargetUploadsUrl, args.GithubPat); + } + + if (args.GitlabServerUrl.HasValue()) + { + var gitlabApiFactory = sp.GetRequiredService(); + + gitlabApi = args.Kerberos + ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + } + + if (args.SshUser.HasValue() || args.SmbUser.HasValue()) + { + var bbsArchiveDownloaderFactory = sp.GetRequiredService(); + var bbsHost = args.ArchiveDownloadHost.HasValue() ? args.ArchiveDownloadHost : new Uri(args.GitlabServerUrl).Host; + + bbsArchiveDownloader = args.SshUser.HasValue() + ? bbsArchiveDownloaderFactory.CreateSshDownloader(bbsHost, args.SshUser, args.SshPrivateKey, args.SshPort, args.GitlabSharedHome) + : bbsArchiveDownloaderFactory.CreateSmbDownloader(bbsHost, args.SmbUser, args.SmbPassword, args.SmbDomain, args.GitlabSharedHome); + } + + var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); + if (azureStorageConnectionString.HasValue()) + { + var azureApiFactory = sp.GetRequiredService(); + azureApi = azureApiFactory.Create(azureStorageConnectionString); + } + + if (args.AwsBucketName.HasValue()) + { + var awsApiFactory = sp.GetRequiredService(); + awsApi = awsApiFactory.Create(args.AwsRegion, args.AwsAccessKey, args.AwsSecretKey, args.AwsSessionToken); + } + + var warningsCountLogger = sp.GetRequiredService(); + + return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, bbsArchiveDownloader, azureApi, awsApi, fileSystemProvider, warningsCountLogger); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs new file mode 100644 index 000000000..533cfb26a --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -0,0 +1,186 @@ +using System.Linq; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgs : CommandArgs +{ + public string ArchiveUrl { get; set; } + public string ArchivePath { get; set; } + + [Secret] + public string AzureStorageConnectionString { get; set; } + + public string AwsBucketName { get; set; } + [Secret] + public string AwsAccessKey { get; set; } + [Secret] + public string AwsSecretKey { get; set; } + [Secret] + public string AwsSessionToken { get; set; } + public string AwsRegion { get; set; } + + public string GithubOrg { get; set; } + public string GithubRepo { get; set; } + [Secret] + public string GithubPat { get; set; } + public bool QueueOnly { get; set; } + public string TargetRepoVisibility { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + public bool Kerberos { get; set; } + + public string GitlabServerUrl { get; set; } + public string GitlabProject { get; set; } + public string GitlabRepo { get; set; } + public string GitlabUsername { get; set; } + [Secret] + public string GitlabPassword { get; set; } + public string GitlabSharedHome { get; set; } + public bool NoSslVerify { get; set; } + + public string ArchiveDownloadHost { get; set; } + public string SshUser { get; set; } + public string SshPrivateKey { get; set; } + public int SshPort { get; set; } = 22; + + public string SmbUser { get; set; } + [Secret] + public string SmbPassword { get; set; } + public string SmbDomain { get; set; } + + public bool KeepArchive { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (!GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue() && !ArchivePath.HasValue()) + { + throw new OctoshiftCliException("Either --bbs-server-url, --archive-path, or --archive-url must be specified."); + } + + if (ArchivePath.HasValue() && ArchiveUrl.HasValue()) + { + throw new OctoshiftCliException("Only one of --archive-path or --archive-url can be specified."); + } + + if (ShouldGenerateArchive()) + { + ValidateGenerateOptions(); + ValidateDownloadOptions(); + } + else + { + ValidateNoGenerateOptions(); + } + + if (ShouldUploadArchive()) + { + ValidateUploadOptions(); + } + + if (ShouldImportArchive()) + { + ValidateImportOptions(); + } + + if (SshPort == 7999) + { + log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); + } + } + + private void ValidateNoGenerateOptions() + { + if (GitlabUsername.HasValue() || GitlabPassword.HasValue()) + { + throw new OctoshiftCliException("--bbs-username and --bbs-password cannot be provided with --archive-path or --archive-url."); + } + + if (NoSslVerify) + { + throw new OctoshiftCliException("--no-ssl-verify cannot be provided with --archive-path or --archive-url."); + } + + if (new[] { SshUser, SshPrivateKey, ArchiveDownloadHost, SmbUser, SmbPassword, SmbDomain }.Any(obj => obj.HasValue())) + { + throw new OctoshiftCliException("SSH or SMB download options cannot be provided with --archive-path or --archive-url."); + } + } + + public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchivePath.HasValue() && !ArchiveUrl.HasValue(); + + public bool ShouldDownloadArchive() => SshUser.HasValue() || SmbUser.HasValue(); + + public bool ShouldUploadArchive() => ArchiveUrl.IsNullOrWhiteSpace() && GithubOrg.HasValue(); + + // NOTE: ArchiveUrl doesn't necessarily refer to the value passed in by the user to the CLI - it is set during CLI runtime when an archive is uploaded to blob storage + public bool ShouldImportArchive() => ArchiveUrl.HasValue() || GithubOrg.HasValue(); + + private void ValidateGenerateOptions() + { + if (Kerberos && (GitlabUsername.HasValue() || GitlabPassword.HasValue())) + { + throw new OctoshiftCliException("--bbs-username and --bbs-password cannot be provided with --kerberos."); + } + + if (GitlabProject.IsNullOrWhiteSpace() || GitlabRepo.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Both --bbs-project and --bbs-repo must be provided."); + } + } + + private void ValidateDownloadOptions() + { + var sshArgs = new[] { SshUser, SshPrivateKey }; + var smbArgs = new[] { SmbUser, SmbPassword }; + var shouldUseSsh = sshArgs.Any(arg => arg.HasValue()); + var shouldUseSmb = smbArgs.Any(arg => arg.HasValue()); + + if (shouldUseSsh && shouldUseSmb) + { + throw new OctoshiftCliException("You can't provide both SSH and SMB credentials together."); + } + + if (SshUser.HasValue() ^ SshPrivateKey.HasValue()) + { + throw new OctoshiftCliException("Both --ssh-user and --ssh-private-key must be specified for SSH download."); + } + + if (ArchiveDownloadHost.HasValue() && !shouldUseSsh && !shouldUseSmb) + { + throw new OctoshiftCliException("--archive-download-host can only be provided if SSH or SMB download options are provided."); + } + } + + private void ValidateUploadOptions() + { + if (AwsBucketName.IsNullOrWhiteSpace() && new[] { AwsAccessKey, AwsSecretKey, AwsSessionToken, AwsRegion }.Any(x => x.HasValue())) + { + throw new OctoshiftCliException("The AWS S3 bucket name must be provided with --aws-bucket-name if other AWS S3 upload options are set."); + } + if (UseGithubStorage && AwsBucketName.HasValue()) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + if (AzureStorageConnectionString.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with a connection string for an Azure storage account. Archive cannot be uploaded to both locations."); + } + } + + private void ValidateImportOptions() + { + if (GithubOrg.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-org must be provided in order to import the Bitbucket archive."); + } + + if (GithubRepo.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-repo must be provided in order to import the Bitbucket archive."); + } + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs new file mode 100644 index 000000000..a6c1792fd --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -0,0 +1,403 @@ +using System; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly GithubApi _githubApi; + private readonly GitlabApi _gitlabApi; + private readonly AzureApi _azureApi; + private readonly AwsApi _awsApi; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly IGitlabArchiveDownloader _bbsArchiveDownloader; + private readonly FileSystemProvider _fileSystemProvider; + private readonly WarningsCountLogger _warningsCountLogger; + private const int CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS = 10000; + private const int CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS = 60000; + + public MigrateRepoCommandHandler( + OctoLogger log, + GithubApi githubApi, + GitlabApi gitlabApi, + EnvironmentVariableProvider environmentVariableProvider, + IGitlabArchiveDownloader bbsArchiveDownloader, + AzureApi azureApi, + AwsApi awsApi, + FileSystemProvider fileSystemProvider, + WarningsCountLogger warningsCountLogger) + { + _log = log; + _githubApi = githubApi; + _gitlabApi = gitlabApi; + _azureApi = azureApi; + _awsApi = awsApi; + _environmentVariableProvider = environmentVariableProvider; + _bbsArchiveDownloader = bbsArchiveDownloader; + _fileSystemProvider = fileSystemProvider; + _warningsCountLogger = warningsCountLogger; + } + + public async Task Handle(MigrateRepoCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + ValidateOptions(args); + + var exportId = 0L; + var migrationSourceId = ""; + + if (args.ShouldImportArchive()) + { + var targetRepoExists = await _githubApi.DoesRepoExist(args.GithubOrg, args.GithubRepo); + + if (targetRepoExists) + { + throw new OctoshiftCliException($"A repository called {args.GithubOrg}/{args.GithubRepo} already exists"); + } + + migrationSourceId = await CreateMigrationSource(args); + } + + if (args.ShouldGenerateArchive()) + { + exportId = await GenerateArchive(args); + + if (args.ShouldDownloadArchive()) + { + args.ArchivePath = await DownloadArchive(exportId); + } + + if (!args.ShouldDownloadArchive() && args.ShouldUploadArchive()) + { + _log.LogWarning($"You haven't specified --ssh-user or --smb-user, so we assume that you're running the CLI on the Bitbucket instance itself. If you are not running this command on the Bitbucket instance, run this command again with the --ssh-user or --smb-user argument to allow the CLI to download the migration archive from the server."); + } + } + + if (args.ShouldUploadArchive()) + { + // This is for the case where the CLI is being run on the BBS server itself + if (args.ArchivePath.IsNullOrWhiteSpace()) + { + args.ArchivePath = GetSourceExportArchiveAbsolutePath(args.GitlabSharedHome, exportId); + } + + _log.LogInformation($"Archive path: {args.ArchivePath}"); + + try + { + if (args.UseGithubStorage) + { + args.ArchiveUrl = await UploadArchiveToGithub(args.GithubOrg, args.ArchivePath); + } +#pragma warning disable IDE0045 + else if (args.AwsBucketName.HasValue()) +#pragma warning restore IDE0045 + { + args.ArchiveUrl = await UploadArchiveToAws(args.AwsBucketName, args.ArchivePath); + } + else + { + args.ArchiveUrl = await UploadArchiveToAzure(args.ArchivePath); + } + + } + finally + { + if (!args.KeepArchive && args.ShouldDownloadArchive()) + { + DeleteArchive(args.ArchivePath); + } + } + } + + if (args.ShouldImportArchive()) + { + await ImportArchive(args, migrationSourceId, args.ArchiveUrl); + } + } + + private string GetSourceExportArchiveAbsolutePath(string bbsSharedHomeDirectory, long exportId) + { + if (bbsSharedHomeDirectory.IsNullOrWhiteSpace()) + { + bbsSharedHomeDirectory = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS + : GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX; + } + + return IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(bbsSharedHomeDirectory, exportId); + } + + private void DeleteArchive(string path) + { + try + { + _fileSystemProvider.DeleteIfExists(path); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _log.LogWarning($"Couldn't delete the downloaded archive. Error message: \"{ex.Message}\""); + _log.LogVerbose(ex.ToString()); + } + } + + private async Task DownloadArchive(long exportId) + { + _log.LogInformation($"Download archive {exportId} started..."); + var downloadedArchiveFullPath = await _bbsArchiveDownloader.Download(exportId); + _log.LogInformation($"Archive was successfully downloaded at \"{downloadedArchiveFullPath}\"."); + + return downloadedArchiveFullPath; + } + + private async Task GenerateArchive(MigrateRepoCommandArgs args) + { + var exportId = await _gitlabApi.StartExport(args.GitlabProject, args.GitlabRepo); + + _log.LogInformation($"Export started. Export ID: {exportId}"); + + var (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(exportId); + + while (ExportState.IsInProgress(exportState)) + { + _log.LogInformation($"Export status: {exportState}; {exportProgress}% complete"); + await Task.Delay(CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS); + (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(exportId); + } + + if (ExportState.IsError(exportState)) + { + throw new OctoshiftCliException($"Bitbucket export failed --> State: {exportState}; Message: {exportMessage}"); + } + + _log.LogInformation($"Export completed. Your migration archive should be ready **on your Bitbucket instance** at $BITBUCKET_SHARED_HOME/data/migration/export/Bitbucket_export_{exportId}.tar"); + + return exportId; + } + + private async Task UploadArchiveToAzure(string archivePath) + { + _log.LogInformation("Uploading Archive to Azure..."); + +#pragma warning disable IDE0063 + await using (var archiveData = _fileSystemProvider.OpenRead(archivePath)) +#pragma warning restore IDE0063 + { + var archiveName = GenerateArchiveName(); + var archiveBlobUrl = await _azureApi.UploadToBlob(archiveName, archiveData); + return archiveBlobUrl.ToString(); + } + } + + private string GenerateArchiveName() => $"{Guid.NewGuid()}.tar"; + + private async Task UploadArchiveToAws(string bucketName, string archivePath) + { + _log.LogInformation("Uploading Archive to AWS..."); + + var keyName = GenerateArchiveName(); + var archiveBlobUrl = await _awsApi.UploadToBucket(bucketName, archivePath, keyName); + + return archiveBlobUrl; + } + + private async Task UploadArchiveToGithub(string org, string archivePath) + { + await using var archiveData = _fileSystemProvider.OpenRead(archivePath); + var githubOrgDatabaseId = await _githubApi.GetOrganizationDatabaseId(org); + + _log.LogInformation("Uploading archive to GitHub Storage"); + var keyName = GenerateArchiveName(); + var authenticatedGitArchiveUri = await _githubApi.UploadArchiveToGithubStorage(githubOrgDatabaseId, keyName, archiveData); + + return authenticatedGitArchiveUri; + } + + private async Task CreateMigrationSource(MigrateRepoCommandArgs args) + { + _log.LogInformation("Creating Migration Source..."); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + try + { + return await _githubApi.CreateGitlabMigrationSource(githubOrgId); + } + catch (OctoshiftCliException ex) when (ex.Message.Contains("not have the correct permissions to execute")) + { + var insufficientPermissionsMessage = InsufficientPermissionsMessageGenerator.Generate(args.GithubOrg); + var message = $"{ex.Message}{insufficientPermissionsMessage}"; + throw new OctoshiftCliException(message, ex); + } + } + + private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSourceId, string archiveUrl = null) + { + _log.LogInformation("Importing Archive..."); + + archiveUrl ??= args.ArchiveUrl; + + var bbsRepoUrl = GetGitlabRepoUrl(args); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + string migrationId; + + try + { + migrationId = await _githubApi.StartGitlabMigration(migrationSourceId, bbsRepoUrl, githubOrgId, args.GithubRepo, args.GithubPat, archiveUrl, args.TargetRepoVisibility); + } + catch (OctoshiftCliException ex) when (ex.Message == $"A repository called {args.GithubOrg}/{args.GithubRepo} already exists") + { + _log.LogWarning($"The Org '{args.GithubOrg}' already contains a repository with the name '{args.GithubRepo}'. No operation will be performed"); + return; + } + + if (args.QueueOnly) + { + _log.LogInformation($"A repository migration (ID: {migrationId}) was successfully queued."); + return; + } + + var (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + + while (RepositoryMigrationStatus.IsPending(migrationState)) + { + _log.LogInformation($"Migration in progress (ID: {migrationId}). State: {migrationState}. Waiting 60 seconds..."); + await Task.Delay(CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS); + (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + } + + var migrationLogAvailableMessage = $"Migration log available at {migrationLogUrl} or by running `gh {CliContext.RootCommand} download-logs --github-org {args.GithubOrg} --github-repo {args.GithubRepo}`"; + + if (RepositoryMigrationStatus.IsFailed(migrationState)) + { + _log.LogError($"Migration Failed. Migration ID: {migrationId}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + throw new OctoshiftCliException(failureReason); + } + + _log.LogSuccess($"Migration completed (ID: {migrationId})! State: {migrationState}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + } + + private string GetAwsAccessKey(MigrateRepoCommandArgs args) => args.AwsAccessKey.HasValue() ? args.AwsAccessKey : _environmentVariableProvider.AwsAccessKeyId(false); + + private string GetAwsSecretKey(MigrateRepoCommandArgs args) => args.AwsSecretKey.HasValue() ? args.AwsSecretKey : _environmentVariableProvider.AwsSecretAccessKey(false); + + private string GetAwsRegion(MigrateRepoCommandArgs args) => args.AwsRegion.HasValue() ? args.AwsRegion : _environmentVariableProvider.AwsRegion(false); + + private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => args.AzureStorageConnectionString.HasValue() + ? args.AzureStorageConnectionString + : _environmentVariableProvider.AzureStorageConnectionString(false); + + private string GetGitlabUsername(MigrateRepoCommandArgs args) => args.GitlabUsername.HasValue() ? args.GitlabUsername : _environmentVariableProvider.GitlabUsername(false); + + private string GetGitlabPassword(MigrateRepoCommandArgs args) => args.GitlabPassword.HasValue() ? args.GitlabPassword : _environmentVariableProvider.GitlabPassword(false); + + private string GetSmbPassword(MigrateRepoCommandArgs args) => args.SmbPassword.HasValue() ? args.SmbPassword : _environmentVariableProvider.SmbPassword(false); + + private string GetGitlabRepoUrl(MigrateRepoCommandArgs args) + { + return args.GitlabServerUrl.HasValue() && args.GitlabProject.HasValue() && args.GitlabRepo.HasValue() + ? $"{args.GitlabServerUrl.TrimEnd('/')}/projects/{args.GitlabProject}/repos/{args.GitlabRepo}/browse" + : "https://not-used"; + } + + private void ValidateOptions(MigrateRepoCommandArgs args) + { + if (args.ShouldGenerateArchive()) + { + if (!args.Kerberos) + { + if (GetGitlabUsername(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("BBS username must be either set as BBS_USERNAME environment variable or passed as --bbs-username."); + } + + if (GetGitlabPassword(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("BBS password must be either set as BBS_PASSWORD environment variable or passed as --bbs-password."); + } + } + + if ((args.SmbUser.HasValue() && GetSmbPassword(args).IsNullOrWhiteSpace()) || (args.SmbPassword.HasValue() && args.SmbUser.IsNullOrWhiteSpace())) + { + throw new OctoshiftCliException("Both --smb-user and --smb-password (or SMB_PASSWORD env. variable) must be specified for SMB download."); + } + + // Validate --bbs-shared-home if running on Bitbucket instance (not using SSH/SMB) + if (!args.ShouldDownloadArchive() && args.GitlabSharedHome.HasValue() && !_fileSystemProvider.DirectoryExists(args.GitlabSharedHome)) + { + throw new OctoshiftCliException($"The path provided for --bbs-shared-home does not exist or is not accessible: {args.GitlabSharedHome}"); + } + } + + // Validate --archive-path if provided + if (args.ArchivePath.HasValue() && !_fileSystemProvider.FileExists(args.ArchivePath)) + { + throw new OctoshiftCliException($"The archive file provided with --archive-path does not exist or is not accessible: {args.ArchivePath}"); + } + + if (args.ShouldUploadArchive()) + { + ValidateUploadOptions(args); + } + } + + private void ValidateUploadOptions(MigrateRepoCommandArgs args) + { + var shouldUseAzureStorage = GetAzureStorageConnectionString(args).HasValue(); + var shouldUseAwsS3 = args.AwsBucketName.HasValue(); + if (!shouldUseAzureStorage && !shouldUseAwsS3 && !args.UseGithubStorage) + { + throw new OctoshiftCliException( + "Either Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) or " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) or " + + "GitHub Storage Option (--use-github-storage) " + + "must be provided."); + } + + if (shouldUseAzureStorage && shouldUseAwsS3) + { + throw new OctoshiftCliException( + "Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) and " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) cannot be " + + "specified together."); + } + + if (shouldUseAwsS3) + { + if (!GetAwsAccessKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-access-key or AWS_ACCESS_KEY_ID environment variable must be set."); + } + + if (!GetAwsSecretKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-secret-key or AWS_SECRET_ACCESS_KEY environment variable must be set."); + } + + if (GetAwsRegion(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Either --aws-region or AWS_REGION environment variable must be set."); + } + } + } +} diff --git a/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs new file mode 100644 index 000000000..587ed556c --- /dev/null +++ b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.ReclaimMannequin; + +namespace OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; + +public sealed class ReclaimMannequinCommand : ReclaimMannequinCommandBase +{ + public ReclaimMannequinCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs new file mode 100644 index 000000000..b94e13975 --- /dev/null +++ b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.RevokeMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; + +public sealed class RevokeMigratorRoleCommand : RevokeMigratorRoleCommandBase +{ + public RevokeMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs new file mode 100644 index 000000000..0a03bd1a9 --- /dev/null +++ b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs @@ -0,0 +1,7 @@ +using OctoshiftCLI.Commands.WaitForMigration; +namespace OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; + +public sealed class WaitForMigrationCommand : WaitForMigrationCommandBase +{ + public WaitForMigrationCommand() => AddOptions(); +} diff --git a/src/gl2gh/ExportState.cs b/src/gl2gh/ExportState.cs new file mode 100644 index 000000000..a406304e2 --- /dev/null +++ b/src/gl2gh/ExportState.cs @@ -0,0 +1,12 @@ +namespace OctoshiftCLI.GitlabToGithub; + +public static class ExportState +{ + public const string COMPLETED = "COMPLETED"; + public const string FAILED = "FAILED"; + public const string ABORTED = "ABORTED"; + + public static bool IsInProgress(string state) => state is not COMPLETED && !IsError(state); + + public static bool IsError(string state) => state is FAILED or ABORTED; +} diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs new file mode 100644 index 000000000..0ce350446 --- /dev/null +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -0,0 +1,44 @@ +using System.Net.Http; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabApiFactory +{ + private readonly OctoLogger _octoLogger; + private readonly IHttpClientFactory _clientFactory; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly IVersionProvider _versionProvider; + private readonly RetryPolicy _retryPolicy; + + public GitlabApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory, EnvironmentVariableProvider environmentVariableProvider, IVersionProvider versionProvider, RetryPolicy retryPolicy) + { + _octoLogger = octoLogger; + _clientFactory = clientFactory; + _environmentVariableProvider = environmentVariableProvider; + _versionProvider = versionProvider; + _retryPolicy = retryPolicy; + } + + public virtual GitlabApi Create(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSsl = false) + { + bbsUsername ??= _environmentVariableProvider.GitlabUsername(); + bbsPassword ??= _environmentVariableProvider.GitlabPassword(); + + var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); + + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("Bitbucket Server"); + var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, bbsUsername, bbsPassword); + return new GitlabApi(bbsClient, bbsServerUrl, _octoLogger); + } + + public virtual GitlabApi CreateKerberos(string bbsServerUrl, bool noSsl = false) + { + var httpClient = noSsl ? _clientFactory.CreateClient("KerberosNoSSL") : _clientFactory.CreateClient("Kerberos"); + + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("Bitbucket Server"); + var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy); + return new GitlabApi(bbsClient, bbsServerUrl, _octoLogger); + } +} diff --git a/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs b/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs new file mode 100644 index 000000000..1e498704f --- /dev/null +++ b/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs @@ -0,0 +1,30 @@ +using OctoshiftCLI.GitlabToGithub.Services; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabArchiveDownloaderFactory +{ + private readonly OctoLogger _log; + private readonly FileSystemProvider _fileSystemProvider; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + + public GitlabArchiveDownloaderFactory(OctoLogger log, FileSystemProvider fileSystemProvider, EnvironmentVariableProvider environmentVariableProvider) + { + _log = log; + _fileSystemProvider = fileSystemProvider; + _environmentVariableProvider = environmentVariableProvider; + } + + public virtual IGitlabArchiveDownloader CreateSshDownloader(string host, string sshUser, string privateKeyFileFullPath, int sshPort = 22, string bbsSharedHomeDirectory = null) => + new GitlabSshArchiveDownloader(_log, _fileSystemProvider, host, sshUser, privateKeyFileFullPath, sshPort) + { + GitlabSharedHomeDirectory = bbsSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX + }; + + public virtual IGitlabArchiveDownloader CreateSmbDownloader(string host, string smbUser, string smbPassword, string domainName = null, string bbsSharedHomeDirectory = null) => + new GitlabSmbArchiveDownloader(_log, _fileSystemProvider, host, smbUser, smbPassword ?? _environmentVariableProvider.SmbPassword(), domainName) + { + GitlabSharedHomeDirectory = bbsSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS + }; +} diff --git a/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs new file mode 100644 index 000000000..131aef62b --- /dev/null +++ b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs @@ -0,0 +1,18 @@ +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabInspectorServiceFactory +{ + private readonly OctoLogger _octoLogger; + private GitlabInspectorService _instance; + + public GitlabInspectorServiceFactory(OctoLogger octoLogger) => _octoLogger = octoLogger; + + public virtual GitlabInspectorService Create(GitlabApi gitlabApi) + { + _instance ??= new(_octoLogger, gitlabApi); + + return _instance; + } +} diff --git a/src/gl2gh/GitlabSettings.cs b/src/gl2gh/GitlabSettings.cs new file mode 100644 index 000000000..0b76fd1a0 --- /dev/null +++ b/src/gl2gh/GitlabSettings.cs @@ -0,0 +1,8 @@ +namespace OctoshiftCLI.GitlabToGithub; + +public static class GitlabSettings +{ + public const string DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS = "c$\\atlassian\\applicationdata\\bitbucket\\shared"; + + public const string DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX = "/var/atlassian/application-data/bitbucket/shared"; +} diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs new file mode 100644 index 000000000..678a380a7 --- /dev/null +++ b/src/gl2gh/Program.cs @@ -0,0 +1,154 @@ +using System; +using System.CommandLine; +using System.CommandLine.Builder; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; + +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.Services; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] +namespace OctoshiftCLI.GitlabToGithub +{ + public static class Program + { + private static readonly OctoLogger Logger = new(); + + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "If the version check fails for any reason, we want the CLI to carry on with the current command")] + public static async Task Main(string[] args) + { + Logger.LogDebug("Execution Started"); + + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(Logger) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddSingleton() + .AddHttpClient("Kerberos", kerberos: true, noSsl: false) + .AddHttpClient("NoSSL", kerberos: false, noSsl: true) + .AddHttpClient("KerberosNoSSL", kerberos: true, noSsl: true) + .AddHttpClient("Default"); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + var rootCommand = new RootCommand("Automate end-to-end GitLab to GitHub migrations.") + .AddCommands(serviceProvider); + + var commandLineBuilder = new CommandLineBuilder(rootCommand); + var parser = commandLineBuilder + .UseDefaults() + .UseExceptionHandler((ex, _) => + { + Logger.LogError(ex); + Environment.ExitCode = 1; + }, 1) + .Build(); + + SetContext(new InvocationContext(parser.Parse(args))); + + try + { + await GithubStatusCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not check GitHub availability from githubstatus.com. See https://www.githubstatus.com for details."); + Logger.LogVerbose(ex.ToString()); + } + + try + { + await LatestVersionCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not retrieve latest gl2gh extension version from github.com, please ensure you are using the latest version by running: gh extension upgrade gl2gh"); + Logger.LogVerbose(ex.ToString()); + } + + await parser.InvokeAsync(args); + } + + private static void SetContext(InvocationContext context) + { + CliContext.RootCommand = "gl2gh"; + CliContext.ExecutingCommand = context.ParseResult.CommandResult.Command.Name; + } + + private static async Task GithubStatusCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipStatusCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped GitHub status check due to GEI_SKIP_STATUS_CHECK environment variable"); + return; + } + + var githubStatusApi = sp.GetRequiredService(); + + if (await githubStatusApi.GetUnresolvedIncidentsCount() > 0) + { + Logger.LogWarning("GitHub is currently experiencing availability issues. See https://www.githubstatus.com for details."); + } + } + + private static async Task LatestVersionCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipVersionCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped latest version check due to GEI_SKIP_VERSION_CHECK environment variable"); + return; + } + + var versionChecker = sp.GetRequiredService(); + + if (await versionChecker.IsLatest()) + { + Logger.LogInformation($"You are running an up-to-date version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]"); + } + else + { + Logger.LogWarning($"You are running an old version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]. The latest version is v{await versionChecker.GetLatestVersion()}."); + Logger.LogWarning($"Please update by running: gh extension upgrade gl2gh"); + } + } + + private static IServiceCollection AddHttpClient(this IServiceCollection serviceCollection, string name, bool kerberos, bool noSsl) => serviceCollection + .AddHttpClient(name) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + UseDefaultCredentials = kerberos, + ServerCertificateCustomValidationCallback = noSsl ? delegate { return true; } : null + }) + .Services; + } +} diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs new file mode 100644 index 000000000..2e080f717 --- /dev/null +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Octoshift.Models; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GitlabInspectorService + { + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + + private IList<(string, string)> _projects; + private readonly IDictionary> _repos = new Dictionary>(); + private readonly IDictionary> _prCounts = new Dictionary>(); + + public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) + { + _log = log; + _gitlabApi = gitlabApi; + } + + public virtual async Task> GetProjects() + { + if (_projects is null) + { + _log.LogInformation($"Retrieving list of all Projects the user has access to..."); + _projects = (await _gitlabApi.GetProjects()) + .Select(project => (project.Key, project.Name)) + .ToList(); + } + + return _projects; + } + + public virtual async Task<(string Key, string Name)> GetProject(string project) + { + _log.LogInformation($"Retrieving Project..."); + var (_, Key, Name) = await _gitlabApi.GetProject(project); + + return (Key, Name); + } + + public virtual async Task> GetRepos(string project) + { + if (!_repos.TryGetValue(project, out var repos)) + { + repos = (await _gitlabApi.GetRepos(project)) + .Select(repo => new GitlabRepository() { Name = repo.Name, Slug = repo.Slug }) + .ToList(); + _repos.Add(project, repos); + } + + return repos; + } + + public virtual async Task GetRepoCount(string[] projects) + { + return await projects.Sum(async key => await GetRepoCount(key)); + } + + public virtual async Task GetRepoCount() + { + var projects = await GetProjects(); + return await projects.Sum(async project => await GetRepoCount(project.Key)); + } + + public virtual async Task GetRepoCount(string project) + { + return (await GetRepos(project)).Count(); + } + + public virtual async Task GetPullRequestCount(string project) + { + var repos = await GetRepos(project); + return await repos.Sum(async repo => await GetRepositoryPullRequestCount(project, repo.Name)); + } + + public virtual async Task GetRepositoryPullRequestCount(string project, string repo) + { + if (!_prCounts.ContainsKey(project)) + { + _prCounts.Add(project, new Dictionary()); + } + + if (!_prCounts[project].TryGetValue(repo, out var prCount)) + { + prCount = (await _gitlabApi.GetRepositoryPullRequests(project, repo)).Count(); + _prCounts[project][repo] = prCount; + } + + return prCount; + } + } +} diff --git a/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs b/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs new file mode 100644 index 000000000..08ff6d6e7 --- /dev/null +++ b/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs @@ -0,0 +1,229 @@ +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using SMBLibrary; +using SMBLibrary.Client; +using FileAttributes = SMBLibrary.FileAttributes; + +namespace OctoshiftCLI.GitlabToGithub.Services; + +public sealed class GitlabSmbArchiveDownloader : IGitlabArchiveDownloader +{ + private const int DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; + + private readonly ISMBClient _smbClient; + private readonly OctoLogger _log; + private readonly FileSystemProvider _fileSystemProvider; + private readonly string _host; + private readonly string _smbUser; + private readonly string _smbPassword; + private readonly string _domainName; + private DateTime _nextProgressReport; + + public GitlabSmbArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, string host, string smbUser, string smbPassword, string domainName = null) + : this(log, fileSystemProvider, new SMB2Client(), host, smbUser, smbPassword, domainName) + { + } + + internal GitlabSmbArchiveDownloader( + OctoLogger log, + FileSystemProvider fileSystemProvider, + ISMBClient smbClient, + string host, + string smbUser, + string smbPassword, + string domainName = null) + { + _log = log ?? throw new ArgumentNullException(nameof(log)); + _fileSystemProvider = fileSystemProvider ?? throw new ArgumentNullException(nameof(fileSystemProvider)); + _smbClient = smbClient ?? throw new ArgumentNullException(nameof(smbClient)); + _host = host ?? throw new ArgumentNullException(nameof(host)); + _smbUser = smbUser; + _smbPassword = smbPassword; + _domainName = domainName; + } + + public string GitlabSharedHomeDirectory { get; init; } = GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS; + + private string GetSourceExportArchiveAbsolutePath(long exportJobId) => + IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(GitlabSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS, exportJobId).ToWindowsPath(); + + public async Task Download(long exportJobId, string targetDirectory = IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY) + { + _nextProgressReport = DateTime.Now; + + ISMBFileStore fileStore = null; + object sourceExportArchiveHandle = null; + + var sourceExportArchiveFullPath = GetSourceExportArchiveAbsolutePath(exportJobId); + var share = sourceExportArchiveFullPath[..sourceExportArchiveFullPath.IndexOf("\\", StringComparison.Ordinal)]; + var sourceExportArchivePathAfterShare = sourceExportArchiveFullPath[(sourceExportArchiveFullPath.IndexOf("\\", StringComparison.Ordinal) + 1)..]; + + var targetExportArchiveFullPath = + Path.Join(targetDirectory ?? IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY, IGitlabArchiveDownloader.GetExportArchiveFileName(exportJobId)).ToUnixPath(); + + await using var targetExportArchive = OpenWriteTargetExportArchive(targetExportArchiveFullPath); + + try + { + ConnectToHost(); + Login(); + + fileStore = CreateSmbFileStore(share); + sourceExportArchiveHandle = CreateFileHandle(fileStore, sourceExportArchivePathAfterShare); + var sourceExportArchiveSize = GetFileSize(fileStore, sourceExportArchiveHandle); + + long bytesRead = 0; + while (true) + { + var status = fileStore.ReadFile(out var data, sourceExportArchiveHandle, bytesRead, (int)_smbClient.MaxReadSize); + + if (IsEndOfFileStatus(status) || data.Length == 0) + { + break; + } + + if (!IsSuccessStatus(status)) + { + throw new OctoshiftCliException($"Failed to read from source export archive \"{sourceExportArchiveFullPath}\" (Status Code: {status})."); + } + + bytesRead += data.Length; + await _fileSystemProvider.WriteAsync(targetExportArchive, data); + + LogProgress(bytesRead, sourceExportArchiveSize); + } + + return targetExportArchiveFullPath; + } + finally + { + if (sourceExportArchiveHandle != null) + { + fileStore?.CloseFile(sourceExportArchiveHandle); + } + + fileStore?.Disconnect(); + _smbClient.Logoff(); + _smbClient.Disconnect(); + } + } + + private void LogProgress(long downloadedBytes, long? totalBytes) + { + if (DateTime.Now < _nextProgressReport) + { + return; + } + + var totalProgressMessage = totalBytes.HasValue + ? $" out of {GetLogFriendlySize(totalBytes.Value)} ({GetPercentage(downloadedBytes, totalBytes.Value)})" + : ""; + _log.LogInformation($"Archive download in progress, {GetLogFriendlySize(downloadedBytes)}{totalProgressMessage} completed..."); + + _nextProgressReport = _nextProgressReport.AddSeconds(DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); + } + + private string GetPercentage(long downloadedBytes, long totalBytes) + { + if (totalBytes is 0L) + { + return "unknown%"; + } + + var percentage = (int)(downloadedBytes * 100D / totalBytes); + return $"{percentage}%"; + } + + private string GetLogFriendlySize(long size) + { + const int kilobyte = 1024; + const int megabyte = 1024 * kilobyte; + const int gigabyte = 1024 * megabyte; + + return size switch + { + < kilobyte => $"{size:n0} bytes", + < megabyte => $"{size / (double)kilobyte:n0} KB", + < gigabyte => $"{size / (double)megabyte:n0} MB", + _ => $"{size / (double)gigabyte:n2} GB" + }; + } + + private FileStream OpenWriteTargetExportArchive(string targetExportArchiveFullPath) + { + _fileSystemProvider.CreateDirectory(Path.GetDirectoryName(targetExportArchiveFullPath)); + return _fileSystemProvider.Open(targetExportArchiveFullPath, FileMode.Create); + } + + private void ConnectToHost() + { + var isConnected = IPAddress.TryParse(_host, out var ipAddress) + ? _smbClient.Connect(ipAddress, SMBTransportType.DirectTCPTransport) + : _smbClient.Connect(_host, SMBTransportType.DirectTCPTransport); + + if (!isConnected) + { + throw new OctoshiftCliException($"Unable to connect to host \"{_host}\"."); + } + } + + private void Login() + { + var status = _smbClient.Login(_domainName ?? "", _smbUser, _smbPassword); + + if (!IsSuccessStatus(status)) + { + throw new OctoshiftCliException($"Unable to login with provided credentials (Status Code: {status})."); + } + } + + private ISMBFileStore CreateSmbFileStore(string shareName) + { + var fileStore = _smbClient.TreeConnect(shareName, out var status); + + return IsSuccessStatus(status) + ? fileStore + : throw new OctoshiftCliException($"Unable to connect to share \"{shareName}\" (Status Code: {status}). " + + "Please make sure that the directory is shared and the share name is correct."); + } + + private object CreateFileHandle(ISMBFileStore fileStore, string sharedFilePath) + { + var status = fileStore.CreateFile( + out var sharedFileHandle, + out _, + sharedFilePath, + AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, + FileAttributes.Normal, + ShareAccess.Read, + CreateDisposition.FILE_OPEN, + CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, + null); + + return IsSuccessStatus(status) + ? sharedFileHandle + : throw new OctoshiftCliException( + $"Couldn't create SMB file handle for \"{sharedFilePath}\" (Status Code: {status})." + + (IsObjectPathNotFoundStatus(status) && GitlabSharedHomeDirectory is GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS + ? "This most likely means that your Bitbucket instance uses a non-default Bitbucket shared home directory, so we couldn't find your archive. " + + "You can point the CLI to a non-default shared directory by specifying the --bbs-shared-home option." + : "")); + } + + private long? GetFileSize(ISMBFileStore fileStore, object sharedFileHandle) + { + var status = fileStore.GetFileInformation(out var fileInfo, sharedFileHandle, FileInformationClass.FileStandardInformation); + + return !IsSuccessStatus(status) ? null : (fileInfo as FileStandardInformation)?.AllocationSize; + } + + private bool IsSuccessStatus(NTStatus status) => status is NTStatus.STATUS_SUCCESS; + + private bool IsEndOfFileStatus(NTStatus status) => status is NTStatus.STATUS_END_OF_FILE; + + private bool IsObjectPathNotFoundStatus(NTStatus status) => status is NTStatus.STATUS_OBJECT_PATH_NOT_FOUND; +} diff --git a/src/gl2gh/Services/GitlabSshArchiveDownloader.cs b/src/gl2gh/Services/GitlabSshArchiveDownloader.cs new file mode 100644 index 000000000..a135227d9 --- /dev/null +++ b/src/gl2gh/Services/GitlabSshArchiveDownloader.cs @@ -0,0 +1,127 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Renci.SshNet; + +namespace OctoshiftCLI.GitlabToGithub.Services; + +public sealed class GitlabSshArchiveDownloader : IGitlabArchiveDownloader, IDisposable +{ + private const int DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; + + private readonly ISftpClient _sftpClient; + private readonly PrivateKeyFile _privateKey; + private readonly OctoLogger _log; + private readonly FileSystemProvider _fileSystemProvider; + private readonly object _mutex = new(); + private DateTime _nextProgressReport; + + public GitlabSshArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, string host, string sshUser, string privateKeyFileFullPath, int sshPort = 22) + { + _log = log; + _fileSystemProvider = fileSystemProvider; + _privateKey = new PrivateKeyFile(privateKeyFileFullPath); + _sftpClient = new SftpClient(host, sshPort, sshUser, _privateKey); + } + + internal GitlabSshArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, ISftpClient sftpClient) + { + _log = log; + _fileSystemProvider = fileSystemProvider; + _sftpClient = sftpClient; + } + + public string GitlabSharedHomeDirectory { get; init; } = GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX; + + private string GetSourceExportArchiveAbsolutePath(long exportJobId) => + IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(GitlabSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX, exportJobId).ToUnixPath(); + + public async Task Download(long exportJobId, string targetDirectory = IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY) + { + _nextProgressReport = DateTime.Now; + + var sourceExportArchiveFullPath = GetSourceExportArchiveAbsolutePath(exportJobId); + var targetExportArchiveFullPath = + Path.Join(targetDirectory ?? IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY, IGitlabArchiveDownloader.GetExportArchiveFileName(exportJobId)).ToUnixPath(); + + if (_sftpClient is BaseClient { IsConnected: false } client) + { + client.Connect(); + } + + if (!_sftpClient.Exists(sourceExportArchiveFullPath)) + { + throw new OctoshiftCliException( + $"Source export archive ({sourceExportArchiveFullPath}) does not exist." + + (GitlabSharedHomeDirectory is GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX + ? "This most likely means that your Bitbucket instance uses a non-default Bitbucket shared home directory, so we couldn't find your archive. " + + "You can point the CLI to a non-default shared directory by specifying the --bbs-shared-home option." + : "")); + } + + _fileSystemProvider.CreateDirectory(targetDirectory); + + var sourceExportArchiveSize = _sftpClient.GetAttributes(sourceExportArchiveFullPath)?.Size ?? long.MaxValue; + await using var targetExportArchive = _fileSystemProvider.Open(targetExportArchiveFullPath, FileMode.Create); + await Task.Factory.FromAsync( + _sftpClient.BeginDownloadFile( + sourceExportArchiveFullPath, + targetExportArchive, + null, + null, + downloaded => LogProgress(downloaded, (ulong)sourceExportArchiveSize)), + _sftpClient.EndDownloadFile); + + return targetExportArchiveFullPath; + } + + private void LogProgress(ulong downloadedBytes, ulong totalBytes) + { + lock (_mutex) + { + if (DateTime.Now < _nextProgressReport) + { + return; + } + + _log.LogInformation( + $"Archive download in progress, {GetLogFriendlySize(downloadedBytes)} out of {GetLogFriendlySize(totalBytes)} ({GetPercentage(downloadedBytes, totalBytes)}) completed..."); + + _nextProgressReport = _nextProgressReport.AddSeconds(DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); + } + } + + private string GetPercentage(ulong downloadedBytes, ulong totalBytes) + { + if (totalBytes is ulong.MinValue) + { + return "unknown%"; + } + + var percentage = (int)(downloadedBytes * 100D / totalBytes); + return $"{percentage}%"; + } + + private string GetLogFriendlySize(ulong size) + { + const int kilobyte = 1024; + const int megabyte = 1024 * kilobyte; + const int gigabyte = 1024 * megabyte; + + return size switch + { + < kilobyte => $"{size:n0} bytes", + < megabyte => $"{size / (double)kilobyte:n0} KB", + < gigabyte => $"{size / (double)megabyte:n0} MB", + _ => $"{size / (double)gigabyte:n2} GB" + }; + } + + public void Dispose() + { + _sftpClient?.Dispose(); + _privateKey?.Dispose(); + } +} diff --git a/src/gl2gh/Services/IBbsArchiveDownloader.cs b/src/gl2gh/Services/IBbsArchiveDownloader.cs new file mode 100644 index 000000000..e929e11c7 --- /dev/null +++ b/src/gl2gh/Services/IBbsArchiveDownloader.cs @@ -0,0 +1,20 @@ +using System.IO; +using System.Threading.Tasks; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.GitlabToGithub.Services; + +public interface IGitlabArchiveDownloader +{ + const string EXPORT_ARCHIVE_SOURCE_DIRECTORY = "data/migration/export"; + const string DEFAULT_TARGET_DIRECTORY = "bbs_archive_downloads"; + + Task Download(long exportJobId, string targetDirectory = DEFAULT_TARGET_DIRECTORY); + + static string GetSourceExportArchiveAbsolutePath(string bbsSharedHomeDirectory, long exportJobId) => + Path.Join(bbsSharedHomeDirectory, GetSourceExportArchiveRelativePath(exportJobId)).ToUnixPath(); + + static string GetExportArchiveFileName(long exportJobId) => $"Bitbucket_export_{exportJobId}.tar"; + + static string GetSourceExportArchiveRelativePath(long exportJobId) => Path.Join(EXPORT_ARCHIVE_SOURCE_DIRECTORY, GetExportArchiveFileName(exportJobId)).ToUnixPath(); +} diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs new file mode 100644 index 000000000..85b760a17 --- /dev/null +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -0,0 +1,47 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class ProjectsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _bbsInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory bbsInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _bbsInspectorServiceFactory = bbsInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSslVerify, string bbsProject = "", bool minimal = false) + { + bbsServerUrl = bbsServerUrl ?? throw new ArgumentNullException(nameof(bbsServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(bbsServerUrl, bbsUsername, bbsPassword, noSslVerify); + var inspector = _bbsInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("project-key,project-name,url,repo-count"); + result.AppendLine(!minimal ? ",pr-count" : null); + + var projects = string.IsNullOrWhiteSpace(bbsProject) ? await inspector.GetProjects() : new[] { await inspector.GetProject(bbsProject) }; + + foreach (var (Key, Name) in projects) + { + var url = $"{bbsServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(Key)}"; + var repoCount = await inspector.GetRepoCount(Key); + var prCount = !minimal ? await inspector.GetPullRequestCount(Key) : 0; + + var projectName = Name.Replace(",", Uri.EscapeDataString(",")); + + result.Append($"\"{Key}\",\"{projectName}\",\"{url}\",{repoCount}"); + result.AppendLine(!minimal ? $",{prCount}" : null); + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/Services/ReposCsvGeneratorService.cs b/src/gl2gh/Services/ReposCsvGeneratorService.cs new file mode 100644 index 000000000..54e31a07c --- /dev/null +++ b/src/gl2gh/Services/ReposCsvGeneratorService.cs @@ -0,0 +1,70 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class ReposCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _bbsInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public ReposCsvGeneratorService(GitlabInspectorServiceFactory bbsInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _bbsInspectorServiceFactory = bbsInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSslVerify, string bbsProject = "", bool minimal = false) + { + bbsServerUrl = bbsServerUrl ?? throw new ArgumentNullException(nameof(bbsServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(bbsServerUrl, bbsUsername, bbsPassword, noSslVerify); + var inspector = _bbsInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"); + result.AppendLine(!minimal ? ",is-archived,pr-count" : null); + + var projects = string.IsNullOrWhiteSpace(bbsProject) ? await inspector.GetProjects() : new[] { await inspector.GetProject(bbsProject) }; + + foreach (var (projectKey, projectName) in projects) + { + foreach (var repo in await inspector.GetRepos(projectKey)) + { + var url = $"{bbsServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(projectKey)}/repos/{Uri.EscapeDataString(repo.Slug)}"; + var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(projectKey, repo.Slug); + var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(projectKey, repo.Slug, bbsUsername, bbsPassword); + var prCount = !minimal ? await inspector.GetRepositoryPullRequestCount(projectKey, repo.Slug) : 0; + + var project = projectName.Replace(",", Uri.EscapeDataString(",")); + var repoName = repo.Name.Replace(",", Uri.EscapeDataString(",")); + + if (lastCommitDate == null) + { + result.Append($"\"{projectKey}\",\"{project}\",\"{repoName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\""); + } + else + { + result.Append($"\"{projectKey}\",\"{project}\",\"{repoName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\""); + } + + try + { + var archived = !minimal && await gitlabApi.GetIsRepositoryArchived(projectKey, repo.Slug); + result.AppendLine(!minimal ? $",\"{archived}\",{prCount}" : null); + } + catch (ArgumentNullException) + { + // The archived field was introduced in BBS 6.0.0 + result.Replace(",is-archived", null); + result.AppendLine(!minimal ? $",{prCount}" : null); + } + } + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/gl2gh.csproj b/src/gl2gh/gl2gh.csproj new file mode 100644 index 000000000..af534add2 --- /dev/null +++ b/src/gl2gh/gl2gh.csproj @@ -0,0 +1,25 @@ + + + + Exe + net8.0 + gl2gh + 12 + OctoshiftCLI.GitlabToGithub + + + + + + + + + + + + + + + + + From b9e52ace654a93cd2aab9a20942195230f14db46 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 15:52:19 -0700 Subject: [PATCH 02/71] Update GitlabApi to use GitLab API routes. --- src/Octoshift/Services/GitlabApi.cs | 142 +++++++++++++--------------- 1 file changed, 67 insertions(+), 75 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index 82d174d57..6f1e9a0b7 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Newtonsoft.Json.Linq; using OctoshiftCLI.Extensions; @@ -24,125 +22,119 @@ public GitlabApi(GitlabClient client, string gitlabServerUrl, OctoLogger log) public virtual async Task GetServerVersion() { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/application-properties"; + var url = $"{_gitlabBaseUrl}/api/v4/version"; var content = await _client.GetAsync(url); return (string)JObject.Parse(content)["version"]; } - public virtual async Task StartExport(string projectKey, string slug) + public virtual async Task StartExport(string groupPath, string repoSlug) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/migration/exports"; - var payload = new - { - repositoriesRequest = new - { - includes = new[] - { - new - { - projectKey, - slug - } - } - } - }; - - var content = await _client.PostAsync(url, payload); - - return (long)JObject.Parse(content)["id"]; + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; + + var exportResponse = await _client.PostAsync(url, new { }); + var exportData = JObject.Parse(exportResponse); + + return (long)exportData["id"]; } - public virtual async Task<(string State, string Message, int Percentage)> GetExport(long id) + public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string repoSlug) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/migration/exports/{id}"; + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; - var content = await _client.GetAsync(url); - var data = JObject.Parse(content); + var exportResponse = await _client.GetAsync(url); + var exportData = JObject.Parse(exportResponse); return ( - (string)data["state"], - (string)data["progress"]["message"], - (int)data["progress"]["percentage"] + (string)exportData["export_status"], + (string)exportData["message"], + (string)exportData["_links"]?["api_url"] ); } - public virtual async Task> GetProjects() + public virtual async Task> GetProjects(string groupPath) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects"; + var encodedGroupPath = Uri.EscapeDataString(groupPath); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?simple=true&per_page=100"; + return await _client.GetAllAsync(url) - .Select(x => ((int)x["id"], (string)x["key"], (string)x["name"])) + .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"])) .ToListAsync(); } - public virtual async Task<(int Id, string Key, string Name)> GetProject(string projectKey) + public virtual async Task<(long Id, string FullPath, string Name)> GetGroup(string groupPath) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}"; - var response = await _client.GetAsync(url); + var encodedGroupPath = groupPath.EscapeDataString(); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}"; - var project = JObject.Parse(response); - return ((int)project["id"], (string)project["key"], (string)project["name"]); + var groupResponse = await _client.GetAsync(url); + var groupData = JObject.Parse(groupResponse); + + return ( + (long)groupData["id"], + (string)groupData["full_path"], + (string)groupData["name"] + ); } - public virtual async Task> GetRepos(string projectKey) + public virtual async Task> GetGroups() { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos"; + var url = $"{_gitlabBaseUrl}/api/v4/groups?per_page=100"; + return await _client.GetAllAsync(url) - .Select(x => ((int)x["id"], (string)x["slug"], (string)x["name"])) + .Select(x => ((long)x["id"], (string)x["full_path"], (string)x["name"])) .ToListAsync(); } - public virtual async Task GetIsRepositoryArchived(string projectKey, string repo) + public virtual async Task GetIsProjectArchived(string groupPath, string repoSlug) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}?fields=archived"; - var response = await _client.GetAsync(url); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?simple=true"; - var data = JObject.Parse(response); - return (bool)data["archived"]; - } + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); - public virtual async Task> GetRepositoryPullRequests(string projectKey, string repo) - { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/pull-requests?state=all"; - return await _client.GetAllAsync(url) - .Select(x => ((int)x["id"], (string)x["name"])) - .ToListAsync(); + return (bool)projectData["archived"]; } - public virtual async Task GetRepositoryLatestCommitDate(string projectKey, string repo) + public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string repoSlug) { - var url = $"{_gitlabBaseUrl}/rest/api/1.0/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/commits?limit=1"; - - try - { - var response = await _client.GetAsync(url); - var commit = JObject.Parse(response); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; - if (commit?["values"] == null || !commit["values"].Any()) - { - return null; - } + var commitsResponse = await _client.GetAsync(url); + var commitsData = JArray.Parse(commitsResponse); + var lastCommittedDate = (string)commitsData.First?["committed_date"]; - var authorTimestamp = (long)commit["values"][0]["authorTimestamp"]; - return DateTimeOffset.FromUnixTimeMilliseconds(authorTimestamp).DateTime; - } - catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + if (string.IsNullOrWhiteSpace(lastCommittedDate)) { return null; } + + return DateTimeOffset.Parse(lastCommittedDate); } - public virtual async Task<(ulong repoSize, ulong attachmentsSize)> GetRepositoryAndAttachmentsSize(string projectKey, string repo, string gitlabUsername, string gitlabPassword) + public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string repoSlug) { - var url = $"{_gitlabBaseUrl}/projects/{projectKey.EscapeDataString()}/repos/{repo.EscapeDataString()}/sizes"; - var response = await _client.GetAsync(url); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true"; - var data = JObject.Parse(response); + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); + var projectStatistics = (JObject)projectData["statistics"]; - var repoSize = (ulong)data["repository"]; - var attachmentsSize = (ulong)data["attachments"]; + var repositorySize = (long)projectStatistics["repository_size"]; + var attachmentsSize = (long)projectStatistics["uploads_size"]; - return (repoSize, attachmentsSize); + return (repositorySize, attachmentsSize); + } + + private static string GetEncodedProjectPath(string groupPath, string repoSlug) + { + var pathWithNamespace = $"{groupPath}/{repoSlug}"; + return pathWithNamespace.EscapeDataString(); } } From 1f689b7f5aa58cf88233a1951f63c47f09dc0582 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 16:09:12 -0700 Subject: [PATCH 03/71] Rename GitlabRepository with GitlabProject. --- src/Octoshift/Models/{GitlabRepository.cs => GitlabProject.cs} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/Octoshift/Models/{GitlabRepository.cs => GitlabProject.cs} (82%) diff --git a/src/Octoshift/Models/GitlabRepository.cs b/src/Octoshift/Models/GitlabProject.cs similarity index 82% rename from src/Octoshift/Models/GitlabRepository.cs rename to src/Octoshift/Models/GitlabProject.cs index c322280c9..9d8da0fe0 100644 --- a/src/Octoshift/Models/GitlabRepository.cs +++ b/src/Octoshift/Models/GitlabProject.cs @@ -1,6 +1,6 @@ namespace Octoshift.Models; -public record GitlabRepository +public record GitlabProject { public string Id { get; init; } public string Name { get; init; } From 9990aecce2396ac74e088b943d40c0e61f4645b0 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 16:10:04 -0700 Subject: [PATCH 04/71] Don't pass credentials to GetRepositoryAndAttachmentsSize. --- src/gl2gh/Services/ReposCsvGeneratorService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Services/ReposCsvGeneratorService.cs b/src/gl2gh/Services/ReposCsvGeneratorService.cs index 54e31a07c..0410a712b 100644 --- a/src/gl2gh/Services/ReposCsvGeneratorService.cs +++ b/src/gl2gh/Services/ReposCsvGeneratorService.cs @@ -35,7 +35,7 @@ public virtual async Task Generate(string bbsServerUrl, string bbsUserna { var url = $"{bbsServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(projectKey)}/repos/{Uri.EscapeDataString(repo.Slug)}"; var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(projectKey, repo.Slug); - var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(projectKey, repo.Slug, bbsUsername, bbsPassword); + var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(projectKey, repo.Slug); var prCount = !minimal ? await inspector.GetRepositoryPullRequestCount(projectKey, repo.Slug) : 0; var project = projectName.Replace(",", Uri.EscapeDataString(",")); From 1d3d4891b16b8792491f6bd4758c729b459e03fa Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 16:11:09 -0700 Subject: [PATCH 05/71] Use correct "group" and "project" verbs in GitlabInspectorService. --- src/gl2gh/Services/GitlabInspectorService.cs | 64 ++++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index 2e080f717..91cc32f5e 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -12,8 +12,8 @@ public class GitlabInspectorService private readonly OctoLogger _log; private readonly GitlabApi _gitlabApi; - private IList<(string, string)> _projects; - private readonly IDictionary> _repos = new Dictionary>(); + private IList<(string, string)> _groups; + private readonly IDictionary> _repos = new Dictionary>(); private readonly IDictionary> _prCounts = new Dictionary>(); public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) @@ -22,73 +22,73 @@ public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) _gitlabApi = gitlabApi; } - public virtual async Task> GetProjects() + public virtual async Task> GetGroups() { - if (_projects is null) + if (_groups is null) { - _log.LogInformation($"Retrieving list of all Projects the user has access to..."); - _projects = (await _gitlabApi.GetProjects()) - .Select(project => (project.Key, project.Name)) + _log.LogInformation($"Retrieving list of all Groups the user has access to..."); + _groups = (await _gitlabApi.GetGroups()) + .Select(group => (group.FullPath, group.Name)) .ToList(); } - return _projects; + return _groups; } - public virtual async Task<(string Key, string Name)> GetProject(string project) + public virtual async Task<(string Key, string Name)> GetGroup(string groupPath) { - _log.LogInformation($"Retrieving Project..."); - var (_, Key, Name) = await _gitlabApi.GetProject(project); + _log.LogInformation($"Retrieving Group..."); + var (_, Key, Name) = await _gitlabApi.GetGroup(groupPath); return (Key, Name); } - public virtual async Task> GetRepos(string project) + public virtual async Task> GetProjects(string groupPath) { - if (!_repos.TryGetValue(project, out var repos)) + if (!_repos.TryGetValue(groupPath, out var repos)) { - repos = (await _gitlabApi.GetRepos(project)) - .Select(repo => new GitlabRepository() { Name = repo.Name, Slug = repo.Slug }) + repos = (await _gitlabApi.GetProjects(groupPath)) + .Select(repo => new GitlabProject() { Name = repo.Name, Path = repo.Path }) .ToList(); - _repos.Add(project, repos); + _repos.Add(groupPath, repos); } return repos; } - public virtual async Task GetRepoCount(string[] projects) + public virtual async Task GetProjectCount(string[] groups) { - return await projects.Sum(async key => await GetRepoCount(key)); + return await groups.Sum(async key => await GetProjectCount(key)); } - public virtual async Task GetRepoCount() + public virtual async Task GetProjectCount() { - var projects = await GetProjects(); - return await projects.Sum(async project => await GetRepoCount(project.Key)); + var groups = await GetGroups(); + return await groups.Sum(async group => await GetProjectCount(group.FullPath)); } - public virtual async Task GetRepoCount(string project) + public virtual async Task GetProjectCount(string groupPath) { - return (await GetRepos(project)).Count(); + return (await GetProjects(groupPath)).Count(); } - public virtual async Task GetPullRequestCount(string project) + public virtual async Task GetPullRequestCount(string groupPath) { - var repos = await GetRepos(project); - return await repos.Sum(async repo => await GetRepositoryPullRequestCount(project, repo.Name)); + var repos = await GetProjects(groupPath); + return await repos.Sum(async repo => await GetProjectPullRequestCount(groupPath, repo.Name)); } - public virtual async Task GetRepositoryPullRequestCount(string project, string repo) + public virtual async Task GetProjectPullRequestCount(string groupPath, string repo) { - if (!_prCounts.ContainsKey(project)) + if (!_prCounts.ContainsKey(groupPath)) { - _prCounts.Add(project, new Dictionary()); + _prCounts.Add(groupPath, new Dictionary()); } - if (!_prCounts[project].TryGetValue(repo, out var prCount)) + if (!_prCounts[groupPath].TryGetValue(repo, out var prCount)) { - prCount = (await _gitlabApi.GetRepositoryPullRequests(project, repo)).Count(); - _prCounts[project][repo] = prCount; + prCount = (await _gitlabApi.GetProjectPullRequests(groupPath, repo)).Count(); + _prCounts[groupPath][repo] = prCount; } return prCount; From a8dce150b6c1e05d19328efa7d191e4d044ab9eb Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:23:18 -0700 Subject: [PATCH 06/71] Add GetAsyncHttpResponseMessage. --- src/Octoshift/Services/GitlabClient.cs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs index 7b7d6bb1f..591d0d7d7 100644 --- a/src/Octoshift/Services/GitlabClient.cs +++ b/src/Octoshift/Services/GitlabClient.cs @@ -47,7 +47,8 @@ public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers public virtual async Task GetAsync(string url) { - return await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + return await response.Content.ReadAsStringAsync(); } public virtual async IAsyncEnumerable GetAllAsync(string url) @@ -69,13 +70,26 @@ public virtual async IAsyncEnumerable GetAllAsync(string url) } } - public virtual async Task PostAsync(string url, object body) => await SendAsync(HttpMethod.Post, url, body); + public virtual async Task GetAsyncHttpResponseMessage(string url) + { + return await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + } - public virtual async Task DeleteAsync(string url) => await SendAsync(HttpMethod.Delete, url); + public virtual async Task PostAsync(string url, object body) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Post, url, body)); + return await response.Content.ReadAsStringAsync(); + } + + public virtual async Task DeleteAsync(string url) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Delete, url)); + return await response.Content.ReadAsStringAsync(); + } private async Task GetWithPagination(string url, int start = 0, int limit = DEFAULT_PAGE_SIZE) => await GetAsync(AddPaginationParams(url, start, limit)); - private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) + private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) { _log.LogVerbose($"HTTP {httpMethod}: {url}"); @@ -99,7 +113,7 @@ private async Task SendAsync(HttpMethod httpMethod, string url, object b response.EnsureSuccessStatusCode(); - return content; + return response; } private string AddPaginationParams(string url, int start, int limit) From 44d2fc4d5ba33040fee1925bd560dd194e30a7dd Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:25:08 -0700 Subject: [PATCH 07/71] Use repoPath in GitlabApi. --- src/Octoshift/Services/GitlabApi.cs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index 6f1e9a0b7..cb4d6c375 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -29,9 +29,9 @@ public virtual async Task GetServerVersion() return (string)JObject.Parse(content)["version"]; } - public virtual async Task StartExport(string groupPath, string repoSlug) + public virtual async Task StartExport(string groupPath, string repoPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; var exportResponse = await _client.PostAsync(url, new { }); @@ -40,9 +40,9 @@ public virtual async Task StartExport(string groupPath, string repoSlug) return (long)exportData["id"]; } - public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string repoSlug) + public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string repoPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; var exportResponse = await _client.GetAsync(url); @@ -89,9 +89,9 @@ public virtual async Task StartExport(string groupPath, string repoSlug) .ToListAsync(); } - public virtual async Task GetIsProjectArchived(string groupPath, string repoSlug) + public virtual async Task GetIsProjectArchived(string groupPath, string repoPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?simple=true"; var projectResponse = await _client.GetAsync(url); @@ -100,9 +100,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return (bool)projectData["archived"]; } - public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string repoSlug) + public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string repoPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; var commitsResponse = await _client.GetAsync(url); @@ -117,9 +117,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return DateTimeOffset.Parse(lastCommittedDate); } - public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string repoSlug) + public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string repoPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoSlug); + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true"; var projectResponse = await _client.GetAsync(url); @@ -132,9 +132,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return (repositorySize, attachmentsSize); } - private static string GetEncodedProjectPath(string groupPath, string repoSlug) + private static string GetEncodedProjectPath(string groupPath, string repoPath) { - var pathWithNamespace = $"{groupPath}/{repoSlug}"; + var pathWithNamespace = $"{groupPath}/{repoPath}"; return pathWithNamespace.EscapeDataString(); } } From 85bb0392ef87d5bc800eccb6f5b7f3136238b7b0 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:25:22 -0700 Subject: [PATCH 08/71] Omit simple=true in GitlabApi. --- src/Octoshift/Services/GitlabApi.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index cb4d6c375..c4fe8b5c1 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -58,7 +58,7 @@ public virtual async Task StartExport(string groupPath, string repoPath) public virtual async Task> GetProjects(string groupPath) { var encodedGroupPath = Uri.EscapeDataString(groupPath); - var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?simple=true&per_page=100"; + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?per_page=100"; return await _client.GetAllAsync(url) .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"])) @@ -92,7 +92,7 @@ public virtual async Task StartExport(string groupPath, string repoPath) public virtual async Task GetIsProjectArchived(string groupPath, string repoPath) { var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); - var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?simple=true"; + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}"; var projectResponse = await _client.GetAsync(url); var projectData = JObject.Parse(projectResponse); From e81760529855e3346eaec03cc9c90011014705a4 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:25:34 -0700 Subject: [PATCH 09/71] Add GetMergeRequestCount to GitlabApi. --- src/Octoshift/Services/GitlabApi.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index c4fe8b5c1..2b80644f4 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -132,6 +132,17 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return (repositorySize, attachmentsSize); } + public virtual async Task GetMergeRequestCount(string groupPath, string repoPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/merge_requests?state=all&per_page=1&page=1"; + + var mrResponse = await _client.GetAsyncHttpResponseMessage(url); + var mrTotal = mrResponse.Headers.GetValues("X-Total").Single(); + + return int.Parse(mrTotal); + } + private static string GetEncodedProjectPath(string groupPath, string repoPath) { var pathWithNamespace = $"{groupPath}/{repoPath}"; From 33d8c218e4b280e5a2164caac1bc38906c68be67 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:27:27 -0700 Subject: [PATCH 10/71] Update ReposCsvGeneratorService to use GitLab. --- .../Services/ReposCsvGeneratorService.cs | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/gl2gh/Services/ReposCsvGeneratorService.cs b/src/gl2gh/Services/ReposCsvGeneratorService.cs index 0410a712b..ab0cfafbc 100644 --- a/src/gl2gh/Services/ReposCsvGeneratorService.cs +++ b/src/gl2gh/Services/ReposCsvGeneratorService.cs @@ -7,52 +7,52 @@ namespace OctoshiftCLI.GitlabToGithub { public class ReposCsvGeneratorService { - private readonly GitlabInspectorServiceFactory _bbsInspectorServiceFactory; + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; private readonly GitlabApiFactory _gitlabApiFactory; - public ReposCsvGeneratorService(GitlabInspectorServiceFactory bbsInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + public ReposCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) { - _bbsInspectorServiceFactory = bbsInspectorServiceFactory; + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; _gitlabApiFactory = gitlabApiFactory; } - public virtual async Task Generate(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSslVerify, string bbsProject = "", bool minimal = false) + public virtual async Task Generate(string gitlabServerUrl, string gitlabUsername, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) { - bbsServerUrl = bbsServerUrl ?? throw new ArgumentNullException(nameof(bbsServerUrl)); + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); - var gitlabApi = _gitlabApiFactory.Create(bbsServerUrl, bbsUsername, bbsPassword, noSslVerify); - var inspector = _bbsInspectorServiceFactory.Create(gitlabApi); + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabUsername, gitlabPassword, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); - result.Append("project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"); + result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"); result.AppendLine(!minimal ? ",is-archived,pr-count" : null); - var projects = string.IsNullOrWhiteSpace(bbsProject) ? await inspector.GetProjects() : new[] { await inspector.GetProject(bbsProject) }; + var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; - foreach (var (projectKey, projectName) in projects) + foreach (var (groupPath, groupName) in groups) { - foreach (var repo in await inspector.GetRepos(projectKey)) + foreach (var project in await inspector.GetProjects(groupPath)) { - var url = $"{bbsServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(projectKey)}/repos/{Uri.EscapeDataString(repo.Slug)}"; - var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(projectKey, repo.Slug); - var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(projectKey, repo.Slug); - var prCount = !minimal ? await inspector.GetRepositoryPullRequestCount(projectKey, repo.Slug) : 0; + var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; + var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); + var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); + var prCount = !minimal ? await inspector.GetRepositoryPullRequestCount(groupPath, project.Path) : 0; - var project = projectName.Replace(",", Uri.EscapeDataString(",")); - var repoName = repo.Name.Replace(",", Uri.EscapeDataString(",")); + var group = groupName.Replace(",", Uri.EscapeDataString(",")); + var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); if (lastCommitDate == null) { - result.Append($"\"{projectKey}\",\"{project}\",\"{repoName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\""); + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\""); } else { - result.Append($"\"{projectKey}\",\"{project}\",\"{repoName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\""); + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\""); } try { - var archived = !minimal && await gitlabApi.GetIsRepositoryArchived(projectKey, repo.Slug); + var archived = !minimal && await gitlabApi.GetIsRepositoryArchived(groupPath, project.Path); result.AppendLine(!minimal ? $",\"{archived}\",{prCount}" : null); } catch (ArgumentNullException) From cf1bbd3e74f4d39e61ba06a2bb6e2fc25cc75b73 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:37:55 -0700 Subject: [PATCH 11/71] Use GroupsCsvGeneratorService for GitLab. --- .../InventoryReport/InventoryReportCommand.cs | 2 +- .../InventoryReportCommandHandler.cs | 10 ++-- src/gl2gh/Program.cs | 2 +- .../Services/GroupsCsvGeneratorService.cs | 47 +++++++++++++++++++ .../Services/ProjectsCsvGeneratorService.cs | 47 ------------------- 5 files changed, 54 insertions(+), 54 deletions(-) create mode 100644 src/gl2gh/Services/GroupsCsvGeneratorService.cs delete mode 100644 src/gl2gh/Services/ProjectsCsvGeneratorService.cs diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 5aca971a6..09a2ba843 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -69,7 +69,7 @@ public override InventoryReportCommandHandler BuildHandler(InventoryReportComman var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); var bbsInspectorServiceFactory = sp.GetRequiredService(); var bbsInspectorService = bbsInspectorServiceFactory.Create(gitlabApi); - var projectsCsvGeneratorService = sp.GetRequiredService(); + var projectsCsvGeneratorService = sp.GetRequiredService(); var reposCsvGeneratorService = sp.GetRequiredService(); return new InventoryReportCommandHandler( diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 19a3783fc..a4df6dcbb 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -14,20 +14,20 @@ public class InventoryReportCommandHandler : ICommandHandler() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs new file mode 100644 index 000000000..2b7e3e375 --- /dev/null +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -0,0 +1,47 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GroupsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public GroupsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string gitlabServerUrl, string gitlabUsername, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + { + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabUsername, gitlabPassword, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("project-key,project-name,url,repo-count"); + result.AppendLine(!minimal ? ",pr-count" : null); + + var projects = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; + + foreach (var (Key, Name) in projects) + { + var url = $"{gitlabServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(Key)}"; + var repoCount = await inspector.GetProjectCount(Key); + var prCount = !minimal ? await inspector.GetPullRequestCount(Key) : 0; + + var projectName = Name.Replace(",", Uri.EscapeDataString(",")); + + result.Append($"\"{Key}\",\"{projectName}\",\"{url}\",{repoCount}"); + result.AppendLine(!minimal ? $",{prCount}" : null); + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs deleted file mode 100644 index 85b760a17..000000000 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Text; -using System.Threading.Tasks; -using OctoshiftCLI.GitlabToGithub.Factories; - -namespace OctoshiftCLI.GitlabToGithub -{ - public class ProjectsCsvGeneratorService - { - private readonly GitlabInspectorServiceFactory _bbsInspectorServiceFactory; - private readonly GitlabApiFactory _gitlabApiFactory; - - public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory bbsInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) - { - _bbsInspectorServiceFactory = bbsInspectorServiceFactory; - _gitlabApiFactory = gitlabApiFactory; - } - - public virtual async Task Generate(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSslVerify, string bbsProject = "", bool minimal = false) - { - bbsServerUrl = bbsServerUrl ?? throw new ArgumentNullException(nameof(bbsServerUrl)); - - var gitlabApi = _gitlabApiFactory.Create(bbsServerUrl, bbsUsername, bbsPassword, noSslVerify); - var inspector = _bbsInspectorServiceFactory.Create(gitlabApi); - var result = new StringBuilder(); - - result.Append("project-key,project-name,url,repo-count"); - result.AppendLine(!minimal ? ",pr-count" : null); - - var projects = string.IsNullOrWhiteSpace(bbsProject) ? await inspector.GetProjects() : new[] { await inspector.GetProject(bbsProject) }; - - foreach (var (Key, Name) in projects) - { - var url = $"{bbsServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(Key)}"; - var repoCount = await inspector.GetRepoCount(Key); - var prCount = !minimal ? await inspector.GetPullRequestCount(Key) : 0; - - var projectName = Name.Replace(",", Uri.EscapeDataString(",")); - - result.Append($"\"{Key}\",\"{projectName}\",\"{url}\",{repoCount}"); - result.AppendLine(!minimal ? $",{prCount}" : null); - } - - return result.ToString(); - } - } -} From 93fca3d6df2599f6dcc6ebb0bddfb20f4f148adc Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:40:24 -0700 Subject: [PATCH 12/71] Use groups in GitLab InventoryReportCommandHandler. --- .../InventoryReportCommandHandler.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index a4df6dcbb..a915060b0 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -40,26 +40,26 @@ public async Task Handle(InventoryReportCommandArgs args) _log.LogInformation("Creating inventory report..."); - var projectKeys = Array.Empty(); - if (string.IsNullOrWhiteSpace(args.GitlabProject)) + var groupKeys = Array.Empty(); + if (string.IsNullOrWhiteSpace(args.GitlabGroup)) { - _log.LogInformation("Finding Projects..."); - var projects = await _gitlabApi.GetProjects(); - projectKeys = projects.Select(x => x.Key).ToArray(); - _log.LogInformation($"Found {projects.Count()} Projects"); + _log.LogInformation("Finding Groups..."); + var groups = await _gitlabApi.GetGroups(); + groupKeys = groups.Select(x => x.Key).ToArray(); + _log.LogInformation($"Found {groups.Count()} Groups"); } _log.LogInformation("Finding Repos..."); - var repoCount = string.IsNullOrWhiteSpace(args.GitlabProject) ? await _bbsInspectorService.GetRepoCount(projectKeys) : await _bbsInspectorService.GetRepoCount(args.GitlabProject); + var repoCount = string.IsNullOrWhiteSpace(args.GitlabGroup) ? await _bbsInspectorService.GetRepoCount(groupKeys) : await _bbsInspectorService.GetRepoCount(args.GitlabGroup); _log.LogInformation($"Found {repoCount} Repos"); - _log.LogInformation("Generating data for projects.csv..."); - var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabProject, args.Minimal); - await WriteToFile("projects.csv", groupsCsvText); - _log.LogSuccess("projects.csv generated"); + _log.LogInformation("Generating data for groups.csv..."); + var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); + await WriteToFile("groups.csv", groupsCsvText); + _log.LogSuccess("groups.csv generated"); _log.LogInformation("Generating repos.csv..."); - var reposCsvText = await _reposCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabProject, args.Minimal); + var reposCsvText = await _reposCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); await WriteToFile("repos.csv", reposCsvText); _log.LogSuccess("repos.csv generated"); } From 6a6c46c9ee32fa41e1c62bc80bfa4e86d657d6d6 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:42:23 -0700 Subject: [PATCH 13/71] Use projects in GitLab InventoryReportCommandHandler. --- .../InventoryReportCommandHandler.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index a915060b0..5283ace42 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -15,20 +15,20 @@ public class InventoryReportCommandHandler : ICommandHandler Date: Wed, 29 Apr 2026 19:43:40 -0700 Subject: [PATCH 14/71] Use ProjectsCsvGeneratorService for GitLab projects. --- .../Commands/InventoryReport/InventoryReportCommandHandler.cs | 4 ++-- ...sCsvGeneratorService.cs => ProjectsCsvGeneratorService.cs} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/gl2gh/Services/{ReposCsvGeneratorService.cs => ProjectsCsvGeneratorService.cs} (94%) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 5283ace42..7a0d73169 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -15,14 +15,14 @@ public class InventoryReportCommandHandler : ICommandHandler Date: Wed, 29 Apr 2026 19:45:50 -0700 Subject: [PATCH 15/71] Use Path in GitlabProject. --- src/Octoshift/Models/GitlabProject.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Octoshift/Models/GitlabProject.cs b/src/Octoshift/Models/GitlabProject.cs index 9d8da0fe0..488d27036 100644 --- a/src/Octoshift/Models/GitlabProject.cs +++ b/src/Octoshift/Models/GitlabProject.cs @@ -4,5 +4,5 @@ public record GitlabProject { public string Id { get; init; } public string Name { get; init; } - public string Slug { get; init; } + public string Path { get; init; } } From 89d45d25c15ccc58805011364f64f97980ae3423 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:47:50 -0700 Subject: [PATCH 16/71] Use groups and projects in GitLab InventoryReportCommand. --- .../Commands/InventoryReport/InventoryReportCommand.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 09a2ba843..9690f2fed 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -69,15 +69,15 @@ public override InventoryReportCommandHandler BuildHandler(InventoryReportComman var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); var bbsInspectorServiceFactory = sp.GetRequiredService(); var bbsInspectorService = bbsInspectorServiceFactory.Create(gitlabApi); - var projectsCsvGeneratorService = sp.GetRequiredService(); - var reposCsvGeneratorService = sp.GetRequiredService(); + var groupsCsvGeneratorService = sp.GetRequiredService(); + var projectsCsvGeneratorService = sp.GetRequiredService(); return new InventoryReportCommandHandler( log, gitlabApi, bbsInspectorService, - projectsCsvGeneratorService, - reposCsvGeneratorService); + groupsCsvGeneratorService, + projectsCsvGeneratorService); } } } From 5b876ad5768b871b9e3088889e72a7c5172765e5 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:48:50 -0700 Subject: [PATCH 17/71] Replace bbs with gitlab in InventoryReportCommand. --- .../InventoryReport/InventoryReportCommand.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 9690f2fed..f6bb1d7f8 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -13,7 +13,7 @@ public InventoryReportCommand() : base( name: "inventory-report", description: "Generates several CSV files containing lists of BBS projects and repos. Useful for planning large migrations. Personal repositories owned by individual users will not be included." + Environment.NewLine + - "Note: Expects BBS_USERNAME and BBS_PASSWORD env variables or --bbs-username and --bbs-password options to be set.") + "Note: Expects BBS_USERNAME and BBS_PASSWORD env variables or --gitlab-username and --gitlab-password options to be set.") { AddOption(GitlabServerUrl); AddOption(GitlabProject); @@ -25,21 +25,21 @@ public InventoryReportCommand() : base( } public Option GitlabServerUrl { get; } = new( - name: "--bbs-server-url", + name: "--gitlab-server-url", description: "The full URL of the Bitbucket Server/Data Center. E.g. http://bitbucket.contoso.com:7990") { IsRequired = true }; public Option GitlabProject { get; } = new( - name: "--bbs-project", + name: "--gitlab-project", description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); public Option GitlabUsername { get; } = new( - name: "--bbs-username", + name: "--gitlab-username", description: "The Bitbucket username of a user with site admin privileges. If not set will be read from BBS_USERNAME environment variable."); public Option GitlabPassword { get; } = new( - name: "--bbs-password", - description: "The Bitbucket password of the user specified by --bbs-username. If not set will be read from BBS_PASSWORD environment variable."); + name: "--gitlab-password", + description: "The Bitbucket password of the user specified by --gitlab-username. If not set will be read from BBS_PASSWORD environment variable."); public Option NoSslVerify { get; } = new( name: "--no-ssl-verify", @@ -67,15 +67,15 @@ public override InventoryReportCommandHandler BuildHandler(InventoryReportComman var log = sp.GetRequiredService(); var gitlabApiFactory = sp.GetRequiredService(); var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); - var bbsInspectorServiceFactory = sp.GetRequiredService(); - var bbsInspectorService = bbsInspectorServiceFactory.Create(gitlabApi); + var gitlabInspectorServiceFactory = sp.GetRequiredService(); + var gitlabInspectorService = gitlabInspectorServiceFactory.Create(gitlabApi); var groupsCsvGeneratorService = sp.GetRequiredService(); var projectsCsvGeneratorService = sp.GetRequiredService(); return new InventoryReportCommandHandler( log, gitlabApi, - bbsInspectorService, + gitlabInspectorService, groupsCsvGeneratorService, projectsCsvGeneratorService); } From 252fd41b953f466765f463d3388856772be6808e Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:50:26 -0700 Subject: [PATCH 18/71] Use GitlabGroup in GitLab InventoryReportCommand. --- .../Commands/InventoryReport/InventoryReportCommand.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index f6bb1d7f8..0179f3ce1 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -16,7 +16,7 @@ public InventoryReportCommand() : base( "Note: Expects BBS_USERNAME and BBS_PASSWORD env variables or --gitlab-username and --gitlab-password options to be set.") { AddOption(GitlabServerUrl); - AddOption(GitlabProject); + AddOption(GitlabGroup); AddOption(GitlabUsername); AddOption(GitlabPassword); AddOption(NoSslVerify); @@ -29,8 +29,8 @@ public InventoryReportCommand() : base( description: "The full URL of the Bitbucket Server/Data Center. E.g. http://bitbucket.contoso.com:7990") { IsRequired = true }; - public Option GitlabProject { get; } = new( - name: "--gitlab-project", + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); public Option GitlabUsername { get; } = new( From c97b47e6b07be059b005a98bd681b8184e17a6bf Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:51:52 -0700 Subject: [PATCH 19/71] Use GitlabGroup in GitLab InventoryReportCommandArgs. --- .../Commands/InventoryReport/InventoryReportCommandArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs index 3b8565f2d..129b52435 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -5,7 +5,7 @@ namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport public class InventoryReportCommandArgs : CommandArgs { public string GitlabServerUrl { get; set; } - public string GitlabProject { get; set; } + public string GitlabGroup { get; set; } public string GitlabUsername { get; set; } [Secret] public string GitlabPassword { get; set; } From 50d47255fa5d3c41ee6bd5d515c9164a521665d7 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:52:44 -0700 Subject: [PATCH 20/71] Use ProjectsCsvGeneratorService in GitLab Program.cs. --- src/gl2gh/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs index d705e720d..1c60b4465 100644 --- a/src/gl2gh/Program.cs +++ b/src/gl2gh/Program.cs @@ -42,7 +42,7 @@ public static async Task Main(string[] args) .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() .AddSingleton() From 399b91927ca0dfad07d805d898df46569fc48aa4 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:56:54 -0700 Subject: [PATCH 21/71] Use groups in GitLab GenerateScriptCommandHandler. --- .../GenerateScriptCommandHandler.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index b7878ea23..e9e9cef16 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -79,22 +79,22 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(VALIDATE_SMB_PASSWORD); } - var projects = args.GitlabProject.HasValue() - ? [args.GitlabProject] - : (await _gitlabApi.GetProjects()).Select(x => x.Key); + var groups = args.GitlabGroup.HasValue() + ? [args.GitlabGroup] + : (await _gitlabApi.GetGroups()).Select(x => x.Path); - foreach (var projectKey in projects) + foreach (var groupPath in groups) { - _log.LogInformation($"Project: {projectKey}"); + _log.LogInformation($"Group: {groupPath}"); content.AppendLine(); - content.AppendLine($"# =========== Project: {projectKey} ==========="); + content.AppendLine($"# =========== Group: {groupPath} ==========="); - var repos = await _gitlabApi.GetRepos(projectKey); + var repos = await _gitlabApi.GetRepos(groupPath); if (!repos.Any()) { - content.AppendLine("# Skipping this project because it has no git repos."); + content.AppendLine("# Skipping this group because it has no git repos."); continue; } @@ -104,7 +104,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) { _log.LogInformation($" Repo: {repoName}"); - content.AppendLine(Exec(MigrateGithubRepoScript(args, projectKey, repoSlug, true))); + content.AppendLine(Exec(MigrateGithubRepoScript(args, groupPath, repoSlug, true))); } } @@ -115,7 +115,7 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb { var bbsServerUrlOption = $" --bbs-server-url \"{args.GitlabServerUrl}\""; var bbsUsernameOption = args.GitlabUsername.HasValue() ? $" --bbs-username \"{args.GitlabUsername}\"" : ""; - var bbsProjectOption = $" --bbs-project \"{bbsProjectKey}\""; + var bbsProjectOption = $" --bbs-group \"{bbsProjectKey}\""; var bbsRepoOption = $" --bbs-repo \"{bbsRepoSlug}\""; var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; var githubRepoOption = $" --github-repo \"{GetGithubRepoName(bbsProjectKey, bbsRepoSlug)}\""; From 6453e84b79cfd5ed56d08ed027a5f1de6650f0b0 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:58:11 -0700 Subject: [PATCH 22/71] Use GitlabGroup in GitLab GenerateScriptCommandArgs. --- src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 709fd55de..3bc5ee589 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -12,7 +12,7 @@ public class GenerateScriptCommandArgs : CommandArgs public string GitlabUsername { get; set; } [Secret] public string GitlabPassword { get; set; } - public string GitlabProject { get; set; } + public string GitlabGroup { get; set; } public string GitlabSharedHome { get; set; } public string ArchiveDownloadHost { get; set; } public string SshUser { get; set; } From becffbdbee0dfdfd6249baf55ccd63845b8d1cc5 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 19:59:40 -0700 Subject: [PATCH 23/71] Use group paths in GitLab InventoryReportCommandHandler. --- .../InventoryReport/InventoryReportCommandHandler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 7a0d73169..144cf4370 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -40,17 +40,17 @@ public async Task Handle(InventoryReportCommandArgs args) _log.LogInformation("Creating inventory report..."); - var groupKeys = Array.Empty(); + var groupPaths = Array.Empty(); if (string.IsNullOrWhiteSpace(args.GitlabGroup)) { _log.LogInformation("Finding Groups..."); var groups = await _gitlabApi.GetGroups(); - groupKeys = groups.Select(x => x.Key).ToArray(); + groupPaths = groups.Select(x => x.Path).ToArray(); _log.LogInformation($"Found {groups.Count()} Groups"); } _log.LogInformation("Finding Projects..."); - var projectCount = string.IsNullOrWhiteSpace(args.GitlabGroup) ? await _bbsInspectorService.GetProjectCount(groupKeys) : await _bbsInspectorService.GetProjectCount(args.GitlabGroup); + var projectCount = string.IsNullOrWhiteSpace(args.GitlabGroup) ? await _bbsInspectorService.GetProjectCount(groupPaths) : await _bbsInspectorService.GetProjectCount(args.GitlabGroup); _log.LogInformation($"Found {projectCount} Projects"); _log.LogInformation("Generating data for groups.csv..."); From 29ba3550296cd8556411143ca1b182bea8539386 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:01:59 -0700 Subject: [PATCH 24/71] Use merge requests in GitLab ProjectsCsvGeneratorService. --- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index 9c9803ccd..46137f137 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -25,7 +25,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var result = new StringBuilder(); result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"); - result.AppendLine(!minimal ? ",is-archived,pr-count" : null); + result.AppendLine(!minimal ? ",is-archived,mr-count" : null); var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; @@ -36,7 +36,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); - var prCount = !minimal ? await inspector.GetRepositoryPullRequestCount(groupPath, project.Path) : 0; + var mrCount = !minimal ? await inspector.GetRepositoryMergeRequestCount(groupPath, project.Path) : 0; var group = groupName.Replace(",", Uri.EscapeDataString(",")); var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); @@ -53,13 +53,13 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab try { var archived = !minimal && await gitlabApi.GetIsRepositoryArchived(groupPath, project.Path); - result.AppendLine(!minimal ? $",\"{archived}\",{prCount}" : null); + result.AppendLine(!minimal ? $",\"{archived}\",{mrCount}" : null); } catch (ArgumentNullException) { // The archived field was introduced in BBS 6.0.0 result.Replace(",is-archived", null); - result.AppendLine(!minimal ? $",{prCount}" : null); + result.AppendLine(!minimal ? $",{mrCount}" : null); } } } From a5fef912d8a4fccafa4f88d10828a0dbeb8ec4ae Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:08:34 -0700 Subject: [PATCH 25/71] Use Path var in GitLabApi. --- src/Octoshift/Services/GitlabApi.cs | 4 ++-- src/gl2gh/Services/GitlabInspectorService.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index 2b80644f4..fe73da79a 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -65,7 +65,7 @@ public virtual async Task StartExport(string groupPath, string repoPath) .ToListAsync(); } - public virtual async Task<(long Id, string FullPath, string Name)> GetGroup(string groupPath) + public virtual async Task<(long Id, string Path, string Name)> GetGroup(string groupPath) { var encodedGroupPath = groupPath.EscapeDataString(); var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}"; @@ -80,7 +80,7 @@ public virtual async Task StartExport(string groupPath, string repoPath) ); } - public virtual async Task> GetGroups() + public virtual async Task> GetGroups() { var url = $"{_gitlabBaseUrl}/api/v4/groups?per_page=100"; diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index 91cc32f5e..132020c06 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -28,7 +28,7 @@ public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) { _log.LogInformation($"Retrieving list of all Groups the user has access to..."); _groups = (await _gitlabApi.GetGroups()) - .Select(group => (group.FullPath, group.Name)) + .Select(group => (group.Path, group.Name)) .ToList(); } @@ -64,7 +64,7 @@ public virtual async Task GetProjectCount(string[] groups) public virtual async Task GetProjectCount() { var groups = await GetGroups(); - return await groups.Sum(async group => await GetProjectCount(group.FullPath)); + return await groups.Sum(async group => await GetProjectCount(group.Path)); } public virtual async Task GetProjectCount(string groupPath) From f08020d03176a0fc55e969fdf9ecacb40becdec3 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:08:59 -0700 Subject: [PATCH 26/71] Call GetProjects from GitLab GenerateScriptCommandHandler. --- .../Commands/GenerateScript/GenerateScriptCommandHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index e9e9cef16..d0c6f45c9 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -90,7 +90,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(); content.AppendLine($"# =========== Group: {groupPath} ==========="); - var repos = await _gitlabApi.GetRepos(groupPath); + var repos = await _gitlabApi.GetProjects(groupPath); if (!repos.Any()) { From b8c99254b9855ab8c919cc20b3a15b4d659dd2e6 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:11:34 -0700 Subject: [PATCH 27/71] Use GetMergeRequestCount in GitLab ProjectsCsvGeneratorService. --- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index 46137f137..e281682e2 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -36,7 +36,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); - var mrCount = !minimal ? await inspector.GetRepositoryMergeRequestCount(groupPath, project.Path) : 0; + var mrCount = !minimal ? await inspector.GetMergeRequestCount(groupPath, project.Path) : 0; var group = groupName.Replace(",", Uri.EscapeDataString(",")); var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); From d4c8782be711985188a99cec2f5db54a28f21c0b Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:12:21 -0700 Subject: [PATCH 28/71] Use projectPath var in GitlabApi. --- src/Octoshift/Services/GitlabApi.cs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index fe73da79a..a8547419e 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -29,9 +29,9 @@ public virtual async Task GetServerVersion() return (string)JObject.Parse(content)["version"]; } - public virtual async Task StartExport(string groupPath, string repoPath) + public virtual async Task StartExport(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; var exportResponse = await _client.PostAsync(url, new { }); @@ -40,9 +40,9 @@ public virtual async Task StartExport(string groupPath, string repoPath) return (long)exportData["id"]; } - public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string repoPath) + public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; var exportResponse = await _client.GetAsync(url); @@ -89,9 +89,9 @@ public virtual async Task StartExport(string groupPath, string repoPath) .ToListAsync(); } - public virtual async Task GetIsProjectArchived(string groupPath, string repoPath) + public virtual async Task GetIsProjectArchived(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}"; var projectResponse = await _client.GetAsync(url); @@ -100,9 +100,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return (bool)projectData["archived"]; } - public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string repoPath) + public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; var commitsResponse = await _client.GetAsync(url); @@ -117,9 +117,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return DateTimeOffset.Parse(lastCommittedDate); } - public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string repoPath) + public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true"; var projectResponse = await _client.GetAsync(url); @@ -132,9 +132,9 @@ public virtual async Task GetIsProjectArchived(string groupPath, string re return (repositorySize, attachmentsSize); } - public virtual async Task GetMergeRequestCount(string groupPath, string repoPath) + public virtual async Task GetMergeRequestCount(string groupPath, string projectPath) { - var encodedProjectPath = GetEncodedProjectPath(groupPath, repoPath); + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/merge_requests?state=all&per_page=1&page=1"; var mrResponse = await _client.GetAsyncHttpResponseMessage(url); @@ -143,9 +143,9 @@ public virtual async Task GetMergeRequestCount(string groupPath, string rep return int.Parse(mrTotal); } - private static string GetEncodedProjectPath(string groupPath, string repoPath) + private static string GetEncodedProjectPath(string groupPath, string projectPath) { - var pathWithNamespace = $"{groupPath}/{repoPath}"; + var pathWithNamespace = $"{groupPath}/{projectPath}"; return pathWithNamespace.EscapeDataString(); } } From bacdf1b07e1186be3b88dc9c2c37386f16e9f7a9 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:29:48 -0700 Subject: [PATCH 29/71] Use merge requests in GitLab GitlabInspectorService. --- src/gl2gh/Services/GitlabInspectorService.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index 132020c06..362f9bb79 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -14,7 +14,7 @@ public class GitlabInspectorService private IList<(string, string)> _groups; private readonly IDictionary> _repos = new Dictionary>(); - private readonly IDictionary> _prCounts = new Dictionary>(); + private readonly IDictionary> _mrCounts = new Dictionary>(); public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) { @@ -72,26 +72,26 @@ public virtual async Task GetProjectCount(string groupPath) return (await GetProjects(groupPath)).Count(); } - public virtual async Task GetPullRequestCount(string groupPath) + public virtual async Task GetMergeRequestCount(string groupPath) { var repos = await GetProjects(groupPath); - return await repos.Sum(async repo => await GetProjectPullRequestCount(groupPath, repo.Name)); + return await repos.Sum(async repo => await GetProjectMergeRequestCount(groupPath, repo.Name)); } - public virtual async Task GetProjectPullRequestCount(string groupPath, string repo) + public virtual async Task GetProjectMergeRequestCount(string groupPath, string repo) { - if (!_prCounts.ContainsKey(groupPath)) + if (!_mrCounts.ContainsKey(groupPath)) { - _prCounts.Add(groupPath, new Dictionary()); + _mrCounts.Add(groupPath, new Dictionary()); } - if (!_prCounts[groupPath].TryGetValue(repo, out var prCount)) + if (!_mrCounts[groupPath].TryGetValue(repo, out var mrCount)) { - prCount = (await _gitlabApi.GetProjectPullRequests(groupPath, repo)).Count(); - _prCounts[groupPath][repo] = prCount; + mrCount = (await _gitlabApi.GetProjectMergeRequests(groupPath, repo)).Count(); + _mrCounts[groupPath][repo] = mrCount; } - return prCount; + return mrCount; } } } From 48f7a680223d72be1032a090175da69723506efa Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:32:13 -0700 Subject: [PATCH 30/71] Use merge requests in GitLab GroupsCsvGeneratorService. --- src/gl2gh/Services/GroupsCsvGeneratorService.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs index 2b7e3e375..e04f5d4f8 100644 --- a/src/gl2gh/Services/GroupsCsvGeneratorService.cs +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -25,7 +25,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var result = new StringBuilder(); result.Append("project-key,project-name,url,repo-count"); - result.AppendLine(!minimal ? ",pr-count" : null); + result.AppendLine(!minimal ? ",mr-count" : null); var projects = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; @@ -33,12 +33,12 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab { var url = $"{gitlabServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(Key)}"; var repoCount = await inspector.GetProjectCount(Key); - var prCount = !minimal ? await inspector.GetPullRequestCount(Key) : 0; + var mrCount = !minimal ? await inspector.GetMergeRequestCount(Key) : 0; var projectName = Name.Replace(",", Uri.EscapeDataString(",")); result.Append($"\"{Key}\",\"{projectName}\",\"{url}\",{repoCount}"); - result.AppendLine(!minimal ? $",{prCount}" : null); + result.AppendLine(!minimal ? $",{mrCount}" : null); } return result.ToString(); From 693d99ac6f1b3a52443323fe22b8f243d5edb0ec Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:35:12 -0700 Subject: [PATCH 31/71] Call GetProjectMergeRequestCount from GitLab ProjectsCsvGeneratorService. --- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index e281682e2..5906b9484 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -36,7 +36,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); - var mrCount = !minimal ? await inspector.GetMergeRequestCount(groupPath, project.Path) : 0; + var mrCount = !minimal ? await inspector.GetProjectMergeRequestCount(groupPath, project.Path) : 0; var group = groupName.Replace(",", Uri.EscapeDataString(",")); var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); From 890c4c15ae6f50026b04dcf9d04552a229694bd0 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:39:17 -0700 Subject: [PATCH 32/71] Use groups and projects in GitLab MigrateRepoCommandHandler. --- .../Commands/MigrateRepo/MigrateRepoCommandHandler.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index a6c1792fd..140f8c695 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -164,7 +164,7 @@ private async Task DownloadArchive(long exportId) private async Task GenerateArchive(MigrateRepoCommandArgs args) { - var exportId = await _gitlabApi.StartExport(args.GitlabProject, args.GitlabRepo); + var exportId = await _gitlabApi.StartExport(args.GitlabGroup, args.GitlabProject); _log.LogInformation($"Export started. Export ID: {exportId}"); @@ -250,7 +250,7 @@ private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSo archiveUrl ??= args.ArchiveUrl; - var bbsRepoUrl = GetGitlabRepoUrl(args); + var bbsRepoUrl = GetGitlabProjectUrl(args); args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); @@ -313,10 +313,10 @@ private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => a private string GetSmbPassword(MigrateRepoCommandArgs args) => args.SmbPassword.HasValue() ? args.SmbPassword : _environmentVariableProvider.SmbPassword(false); - private string GetGitlabRepoUrl(MigrateRepoCommandArgs args) + private string GetGitlabProjectUrl(MigrateRepoCommandArgs args) { - return args.GitlabServerUrl.HasValue() && args.GitlabProject.HasValue() && args.GitlabRepo.HasValue() - ? $"{args.GitlabServerUrl.TrimEnd('/')}/projects/{args.GitlabProject}/repos/{args.GitlabRepo}/browse" + return args.GitlabServerUrl.HasValue() && args.GitlabGroup.HasValue() && args.GitlabProject.HasValue() + ? $"{args.GitlabServerUrl.TrimEnd('/')}/projects/{args.GitlabGroup}/repos/{args.GitlabProject}/browse" : "https://not-used"; } From 17c8c8d1b3a0f319ccc728f38be1a93d8663ebc7 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:45:22 -0700 Subject: [PATCH 33/71] Consume "archived" value from project API for GitLab. --- src/Octoshift/Models/GitlabProject.cs | 1 + src/Octoshift/Services/GitlabApi.cs | 4 ++-- .../GenerateScript/GenerateScriptCommandHandler.cs | 2 +- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 12 +----------- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/Octoshift/Models/GitlabProject.cs b/src/Octoshift/Models/GitlabProject.cs index 488d27036..10b337d65 100644 --- a/src/Octoshift/Models/GitlabProject.cs +++ b/src/Octoshift/Models/GitlabProject.cs @@ -5,4 +5,5 @@ public record GitlabProject public string Id { get; init; } public string Name { get; init; } public string Path { get; init; } + public string Archived { get; init; } } diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index a8547419e..b2ee4ea04 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -55,13 +55,13 @@ public virtual async Task StartExport(string groupPath, string projectPath ); } - public virtual async Task> GetProjects(string groupPath) + public virtual async Task> GetProjects(string groupPath) { var encodedGroupPath = Uri.EscapeDataString(groupPath); var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?per_page=100"; return await _client.GetAllAsync(url) - .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"])) + .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"], (bool)x["archived"])) .ToListAsync(); } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index d0c6f45c9..93dcbedb9 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -100,7 +100,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(); - foreach (var (_, repoSlug, repoName) in repos) + foreach (var (_, repoSlug, repoName, _) in repos) { _log.LogInformation($" Repo: {repoName}"); diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index 5906b9484..a3fc65134 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -50,17 +50,7 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\""); } - try - { - var archived = !minimal && await gitlabApi.GetIsRepositoryArchived(groupPath, project.Path); - result.AppendLine(!minimal ? $",\"{archived}\",{mrCount}" : null); - } - catch (ArgumentNullException) - { - // The archived field was introduced in BBS 6.0.0 - result.Replace(",is-archived", null); - result.AppendLine(!minimal ? $",{mrCount}" : null); - } + result.AppendLine(!minimal ? $",\"{project.Archived}\",{mrCount}" : null); } } From fa6e445d248c0c51ab6348c9583600bb10660e6e Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:48:18 -0700 Subject: [PATCH 34/71] Use group and project in GitLab MigrateRepoCommandArgs. --- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index 533cfb26a..e4c3495c0 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -33,8 +33,8 @@ public class MigrateRepoCommandArgs : CommandArgs public bool Kerberos { get; set; } public string GitlabServerUrl { get; set; } + public string GitlabGroup { get; set; } public string GitlabProject { get; set; } - public string GitlabRepo { get; set; } public string GitlabUsername { get; set; } [Secret] public string GitlabPassword { get; set; } @@ -126,9 +126,9 @@ private void ValidateGenerateOptions() throw new OctoshiftCliException("--bbs-username and --bbs-password cannot be provided with --kerberos."); } - if (GitlabProject.IsNullOrWhiteSpace() || GitlabRepo.IsNullOrWhiteSpace()) + if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("Both --bbs-project and --bbs-repo must be provided."); + throw new OctoshiftCliException("Both --bbs-group and --bbs-project must be provided."); } } From a6a62f4a1cc430458b2fcdf16050f4341773dec5 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:49:12 -0700 Subject: [PATCH 35/71] Use gitlab options in GitLab MigrateRepoCommandArgs. --- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index e4c3495c0..1a3c38df2 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -58,7 +58,7 @@ public override void Validate(OctoLogger log) { if (!GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue() && !ArchivePath.HasValue()) { - throw new OctoshiftCliException("Either --bbs-server-url, --archive-path, or --archive-url must be specified."); + throw new OctoshiftCliException("Either --gitlab-server-url, --archive-path, or --archive-url must be specified."); } if (ArchivePath.HasValue() && ArchiveUrl.HasValue()) @@ -96,7 +96,7 @@ private void ValidateNoGenerateOptions() { if (GitlabUsername.HasValue() || GitlabPassword.HasValue()) { - throw new OctoshiftCliException("--bbs-username and --bbs-password cannot be provided with --archive-path or --archive-url."); + throw new OctoshiftCliException("--gitlab-username and --gitlab-password cannot be provided with --archive-path or --archive-url."); } if (NoSslVerify) @@ -123,12 +123,12 @@ private void ValidateGenerateOptions() { if (Kerberos && (GitlabUsername.HasValue() || GitlabPassword.HasValue())) { - throw new OctoshiftCliException("--bbs-username and --bbs-password cannot be provided with --kerberos."); + throw new OctoshiftCliException("--gitlab-username and --gitlab-password cannot be provided with --kerberos."); } if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("Both --bbs-group and --bbs-project must be provided."); + throw new OctoshiftCliException("Both --gitlab-group and --gitlab-project must be provided."); } } From 3a2c55a77f955ee15c6bf3ba11d7820078261a27 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 29 Apr 2026 20:50:46 -0700 Subject: [PATCH 36/71] Pass group and project to GetExport from GL MigrateRepoCommandHandler. --- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index 140f8c695..0b5ae88bd 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -168,13 +168,13 @@ private async Task GenerateArchive(MigrateRepoCommandArgs args) _log.LogInformation($"Export started. Export ID: {exportId}"); - var (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(exportId); + var (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); while (ExportState.IsInProgress(exportState)) { _log.LogInformation($"Export status: {exportState}; {exportProgress}% complete"); await Task.Delay(CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS); - (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(exportId); + (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); } if (ExportState.IsError(exportState)) From 6445ece923dcf8ce20615665f2e18b5306d44121 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Mon, 11 May 2026 13:32:22 -0700 Subject: [PATCH 37/71] Return Path from GetGroups in GitlabInspectorService. --- src/gl2gh/Services/GitlabInspectorService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index 362f9bb79..e7a1570bc 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -22,7 +22,7 @@ public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) _gitlabApi = gitlabApi; } - public virtual async Task> GetGroups() + public virtual async Task> GetGroups() { if (_groups is null) { From 6c7702c5a0c9f80263ea5b9d567ca170709af2c3 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Mon, 11 May 2026 13:40:31 -0700 Subject: [PATCH 38/71] Use GetProjectMergeRequestCount in GitlabInspectorService. --- src/gl2gh/Services/GitlabInspectorService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index e7a1570bc..a4f340c1e 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -87,7 +87,7 @@ public virtual async Task GetProjectMergeRequestCount(string groupPath, str if (!_mrCounts[groupPath].TryGetValue(repo, out var mrCount)) { - mrCount = (await _gitlabApi.GetProjectMergeRequests(groupPath, repo)).Count(); + mrCount = await _gitlabApi.GetMergeRequestCount(groupPath, repo); _mrCounts[groupPath][repo] = mrCount; } From ba99ac715804129e517a0abb4af0d6a627246483 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 11:53:01 -0700 Subject: [PATCH 39/71] Remove username from GitLab auth. --- .../Commands/GenerateScript/GenerateScriptCommand.cs | 7 +------ .../Commands/InventoryReport/InventoryReportCommand.cs | 9 ++------- .../InventoryReport/InventoryReportCommandHandler.cs | 4 ++-- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs | 7 +------ src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs | 9 ++++----- .../Commands/MigrateRepo/MigrateRepoCommandHandler.cs | 7 ------- src/gl2gh/Factories/GitlabApiFactory.cs | 5 ++--- src/gl2gh/Services/GroupsCsvGeneratorService.cs | 4 ++-- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 4 ++-- 9 files changed, 16 insertions(+), 40 deletions(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs index 1210a5058..16b456816 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -18,7 +18,6 @@ public GenerateScriptCommand() : base( AddOption(GitlabServerUrl); AddOption(GithubOrg); AddOption(TargetApiUrl); - AddOption(GitlabUsername); AddOption(GitlabPassword); AddOption(GitlabProject); AddOption(GitlabSharedHome); @@ -43,10 +42,6 @@ public GenerateScriptCommand() : base( description: "The full URL of the Bitbucket Server/Data Center to migrate from.") { IsRequired = true }; - public Option GitlabUsername { get; } = new( - name: "--bbs-username", - description: "The Bitbucket username of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_USERNAME environment variable."); - public Option GitlabPassword { get; } = new( name: "--bbs-password", description: "The Bitbucket password of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_PASSWORD environment variable." + @@ -152,7 +147,7 @@ public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandA var gitlabApiFactory = sp.GetRequiredService(); var gitlabApi = args.Kerberos ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi, environmentVariableProvider); } diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 0179f3ce1..7b90535e9 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -13,11 +13,10 @@ public InventoryReportCommand() : base( name: "inventory-report", description: "Generates several CSV files containing lists of BBS projects and repos. Useful for planning large migrations. Personal repositories owned by individual users will not be included." + Environment.NewLine + - "Note: Expects BBS_USERNAME and BBS_PASSWORD env variables or --gitlab-username and --gitlab-password options to be set.") + "Note: Expects GITLAB_PAT env variable or --gitlab-pat options to be set.") { AddOption(GitlabServerUrl); AddOption(GitlabGroup); - AddOption(GitlabUsername); AddOption(GitlabPassword); AddOption(NoSslVerify); AddOption(Minimal); @@ -33,10 +32,6 @@ public InventoryReportCommand() : base( name: "--gitlab-group", description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); - public Option GitlabUsername { get; } = new( - name: "--gitlab-username", - description: "The Bitbucket username of a user with site admin privileges. If not set will be read from BBS_USERNAME environment variable."); - public Option GitlabPassword { get; } = new( name: "--gitlab-password", description: "The Bitbucket password of the user specified by --gitlab-username. If not set will be read from BBS_PASSWORD environment variable."); @@ -66,7 +61,7 @@ public override InventoryReportCommandHandler BuildHandler(InventoryReportComman var log = sp.GetRequiredService(); var gitlabApiFactory = sp.GetRequiredService(); - var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); var gitlabInspectorServiceFactory = sp.GetRequiredService(); var gitlabInspectorService = gitlabInspectorServiceFactory.Create(gitlabApi); var groupsCsvGeneratorService = sp.GetRequiredService(); diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 144cf4370..72bf03327 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -54,12 +54,12 @@ public async Task Handle(InventoryReportCommandArgs args) _log.LogInformation($"Found {projectCount} Projects"); _log.LogInformation("Generating data for groups.csv..."); - var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); + var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); await WriteToFile("groups.csv", groupsCsvText); _log.LogSuccess("groups.csv generated"); _log.LogInformation("Generating projects.csv..."); - var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); + var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); await WriteToFile("projects.csv", projectsCsvText); _log.LogSuccess("projects.csv generated"); } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 78d97f905..748879079 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -26,7 +26,6 @@ public MigrateRepoCommand() : base( AddOption(GitlabServerUrl); AddOption(GitlabProject); AddOption(GitlabRepo); - AddOption(GitlabUsername); AddOption(GitlabPassword); AddOption(GitlabSharedHome); AddOption(SshUser); @@ -75,10 +74,6 @@ public MigrateRepoCommand() : base( IsRequired = true }; - public Option GitlabUsername { get; } = new( - name: "--bbs-username", - description: "The Bitbucket username of a user with site admin privileges. If not set will be read from BBS_USERNAME environment variable."); - public Option GitlabPassword { get; } = new( name: "--bbs-password", description: "The Bitbucket password of the user specified by --bbs-username. If not set will be read from BBS_PASSWORD environment variable."); @@ -238,7 +233,7 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar gitlabApi = args.Kerberos ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabUsername, args.GitlabPassword, args.NoSslVerify); + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); } if (args.SshUser.HasValue() || args.SmbUser.HasValue()) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index 1a3c38df2..3ad44a349 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -35,7 +35,6 @@ public class MigrateRepoCommandArgs : CommandArgs public string GitlabServerUrl { get; set; } public string GitlabGroup { get; set; } public string GitlabProject { get; set; } - public string GitlabUsername { get; set; } [Secret] public string GitlabPassword { get; set; } public string GitlabSharedHome { get; set; } @@ -94,9 +93,9 @@ public override void Validate(OctoLogger log) private void ValidateNoGenerateOptions() { - if (GitlabUsername.HasValue() || GitlabPassword.HasValue()) + if (GitlabPassword.HasValue()) { - throw new OctoshiftCliException("--gitlab-username and --gitlab-password cannot be provided with --archive-path or --archive-url."); + throw new OctoshiftCliException("--gitlab-password cannot be provided with --archive-path or --archive-url."); } if (NoSslVerify) @@ -121,9 +120,9 @@ private void ValidateNoGenerateOptions() private void ValidateGenerateOptions() { - if (Kerberos && (GitlabUsername.HasValue() || GitlabPassword.HasValue())) + if (Kerberos && GitlabPassword.HasValue()) { - throw new OctoshiftCliException("--gitlab-username and --gitlab-password cannot be provided with --kerberos."); + throw new OctoshiftCliException("--gitlab-password cannot be provided with --kerberos."); } if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index 0b5ae88bd..26223526b 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -307,8 +307,6 @@ private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => a ? args.AzureStorageConnectionString : _environmentVariableProvider.AzureStorageConnectionString(false); - private string GetGitlabUsername(MigrateRepoCommandArgs args) => args.GitlabUsername.HasValue() ? args.GitlabUsername : _environmentVariableProvider.GitlabUsername(false); - private string GetGitlabPassword(MigrateRepoCommandArgs args) => args.GitlabPassword.HasValue() ? args.GitlabPassword : _environmentVariableProvider.GitlabPassword(false); private string GetSmbPassword(MigrateRepoCommandArgs args) => args.SmbPassword.HasValue() ? args.SmbPassword : _environmentVariableProvider.SmbPassword(false); @@ -326,11 +324,6 @@ private void ValidateOptions(MigrateRepoCommandArgs args) { if (!args.Kerberos) { - if (GetGitlabUsername(args).IsNullOrWhiteSpace()) - { - throw new OctoshiftCliException("BBS username must be either set as BBS_USERNAME environment variable or passed as --bbs-username."); - } - if (GetGitlabPassword(args).IsNullOrWhiteSpace()) { throw new OctoshiftCliException("BBS password must be either set as BBS_PASSWORD environment variable or passed as --bbs-password."); diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs index 0ce350446..61a51512a 100644 --- a/src/gl2gh/Factories/GitlabApiFactory.cs +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -21,15 +21,14 @@ public GitlabApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory, _retryPolicy = retryPolicy; } - public virtual GitlabApi Create(string bbsServerUrl, string bbsUsername, string bbsPassword, bool noSsl = false) + public virtual GitlabApi Create(string bbsServerUrl, string bbsPassword, bool noSsl = false) { - bbsUsername ??= _environmentVariableProvider.GitlabUsername(); bbsPassword ??= _environmentVariableProvider.GitlabPassword(); var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("Bitbucket Server"); - var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, bbsUsername, bbsPassword); + var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, bbsPassword); return new GitlabApi(bbsClient, bbsServerUrl, _octoLogger); } diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs index e04f5d4f8..be38c9372 100644 --- a/src/gl2gh/Services/GroupsCsvGeneratorService.cs +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -16,11 +16,11 @@ public GroupsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorSe _gitlabApiFactory = gitlabApiFactory; } - public virtual async Task Generate(string gitlabServerUrl, string gitlabUsername, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + public virtual async Task Generate(string gitlabServerUrl, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) { gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); - var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabUsername, gitlabPassword, noSslVerify); + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPassword, noSslVerify); var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index a3fc65134..b84562b8a 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -16,11 +16,11 @@ public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspector _gitlabApiFactory = gitlabApiFactory; } - public virtual async Task Generate(string gitlabServerUrl, string gitlabUsername, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + public virtual async Task Generate(string gitlabServerUrl, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) { gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); - var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabUsername, gitlabPassword, noSslVerify); + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPassword, noSslVerify); var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); From 700b902223f810b453cd57286655a7f0bacb1aac Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 11:57:17 -0700 Subject: [PATCH 40/71] Use GitLab references in GitlabApiFactory. --- src/gl2gh/Factories/GitlabApiFactory.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs index 61a51512a..55dd47046 100644 --- a/src/gl2gh/Factories/GitlabApiFactory.cs +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -21,23 +21,23 @@ public GitlabApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory, _retryPolicy = retryPolicy; } - public virtual GitlabApi Create(string bbsServerUrl, string bbsPassword, bool noSsl = false) + public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPassword, bool noSsl = false) { - bbsPassword ??= _environmentVariableProvider.GitlabPassword(); + gitlabPassword ??= _environmentVariableProvider.GitlabPassword(); var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); - var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("Bitbucket Server"); - var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, bbsPassword); - return new GitlabApi(bbsClient, bbsServerUrl, _octoLogger); + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPassword); + return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } - public virtual GitlabApi CreateKerberos(string bbsServerUrl, bool noSsl = false) + public virtual GitlabApi CreateKerberos(string gitlabServerUrl, bool noSsl = false) { var httpClient = noSsl ? _clientFactory.CreateClient("KerberosNoSSL") : _clientFactory.CreateClient("Kerberos"); - var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("Bitbucket Server"); - var bbsClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy); - return new GitlabApi(bbsClient, bbsServerUrl, _octoLogger); + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy); + return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } } From e5f0c7248c96dc4289dfa1a711bd855a7bae6f26 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 12:13:10 -0700 Subject: [PATCH 41/71] Remove GITLAB_USERNAME env var from EnvironmentVariableProvider. --- src/Octoshift/Services/EnvironmentVariableProvider.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Octoshift/Services/EnvironmentVariableProvider.cs b/src/Octoshift/Services/EnvironmentVariableProvider.cs index f1a3f7d70..50c21837d 100644 --- a/src/Octoshift/Services/EnvironmentVariableProvider.cs +++ b/src/Octoshift/Services/EnvironmentVariableProvider.cs @@ -15,7 +15,6 @@ public class EnvironmentVariableProvider private const string AWS_REGION = "AWS_REGION"; private const string BBS_USERNAME = "BBS_USERNAME"; private const string BBS_PASSWORD = "BBS_PASSWORD"; - private const string GITLAB_USERNAME = "GITLAB_USERNAME"; private const string GITLAB_PASSWORD = "GITLAB_PASSWORD"; private const string SMB_PASSWORD = "SMB_PASSWORD"; private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK"; @@ -59,9 +58,6 @@ public virtual string BbsUsername(bool throwIfNotFound = true) => public virtual string BbsPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); - public virtual string GitlabUsername(bool throwIfNotFound = true) => - GetSecret(BBS_USERNAME, throwIfNotFound); - public virtual string GitlabPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); From 5149a203df816c8778a71216276778b6de64eba2 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 12:18:57 -0700 Subject: [PATCH 42/71] Use PAT args for GitLab. --- .../Services/EnvironmentVariableProvider.cs | 6 +++--- src/Octoshift/Services/GitlabClient.cs | 5 ++--- .../Commands/GenerateScript/GenerateScriptCommand.cs | 12 ++++++------ .../GenerateScript/GenerateScriptCommandArgs.cs | 2 +- .../GenerateScript/GenerateScriptCommandHandler.cs | 10 +++++----- .../InventoryReport/InventoryReportCommand.cs | 10 +++++----- .../InventoryReport/InventoryReportCommandArgs.cs | 2 +- .../InventoryReport/InventoryReportCommandHandler.cs | 4 ++-- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs | 12 ++++++------ .../Commands/MigrateRepo/MigrateRepoCommandArgs.cs | 10 +++++----- .../MigrateRepo/MigrateRepoCommandHandler.cs | 6 +++--- src/gl2gh/Factories/GitlabApiFactory.cs | 6 +++--- src/gl2gh/Services/GroupsCsvGeneratorService.cs | 4 ++-- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 4 ++-- 14 files changed, 46 insertions(+), 47 deletions(-) diff --git a/src/Octoshift/Services/EnvironmentVariableProvider.cs b/src/Octoshift/Services/EnvironmentVariableProvider.cs index 50c21837d..878d92734 100644 --- a/src/Octoshift/Services/EnvironmentVariableProvider.cs +++ b/src/Octoshift/Services/EnvironmentVariableProvider.cs @@ -15,7 +15,7 @@ public class EnvironmentVariableProvider private const string AWS_REGION = "AWS_REGION"; private const string BBS_USERNAME = "BBS_USERNAME"; private const string BBS_PASSWORD = "BBS_PASSWORD"; - private const string GITLAB_PASSWORD = "GITLAB_PASSWORD"; + private const string GITLAB_PAT = "GITLAB_PAT"; private const string SMB_PASSWORD = "SMB_PASSWORD"; private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK"; private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK"; @@ -58,8 +58,8 @@ public virtual string BbsUsername(bool throwIfNotFound = true) => public virtual string BbsPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); - public virtual string GitlabPassword(bool throwIfNotFound = true) => - GetSecret(BBS_PASSWORD, throwIfNotFound); + public virtual string GitlabPat(bool throwIfNotFound = true) => + GetSecret(GITLAB_PAT, throwIfNotFound); public virtual string SmbPassword(bool throwIfNotFound = true) => GetSecret(SMB_PASSWORD, throwIfNotFound); diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs index 591d0d7d7..51af34a56 100644 --- a/src/Octoshift/Services/GitlabClient.cs +++ b/src/Octoshift/Services/GitlabClient.cs @@ -18,13 +18,12 @@ public class GitlabClient private readonly OctoLogger _log; private readonly RetryPolicy _retryPolicy; - public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string username, string password) : + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string gitlabPat) : this(log, httpClient, versionProvider, retryPolicy) { if (_httpClient != null) { - var authCredentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{username}:{password}")); - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", authCredentials); + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("PRIVATE-TOKEN", gitlabPat); } } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs index 16b456816..4b07976d3 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -18,7 +18,7 @@ public GenerateScriptCommand() : base( AddOption(GitlabServerUrl); AddOption(GithubOrg); AddOption(TargetApiUrl); - AddOption(GitlabPassword); + AddOption(GitlabPat); AddOption(GitlabProject); AddOption(GitlabSharedHome); AddOption(SshUser); @@ -42,11 +42,11 @@ public GenerateScriptCommand() : base( description: "The full URL of the Bitbucket Server/Data Center to migrate from.") { IsRequired = true }; - public Option GitlabPassword { get; } = new( - name: "--bbs-password", - description: "The Bitbucket password of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_PASSWORD environment variable." + + public Option GitlabPat { get; } = new( + name: "--bbs-pat", + description: "The Bitbucket PAT of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_PASSWORD environment variable." + $"{Environment.NewLine}" + - "Note: The password will not get included in the generated script and it has to be set as an env variable before running the script."); + "Note: The PAT will not get included in the generated script and it has to be set as an env variable before running the script."); public Option GitlabProject { get; } = new( name: "--bbs-project", @@ -147,7 +147,7 @@ public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandA var gitlabApiFactory = sp.GetRequiredService(); var gitlabApi = args.Kerberos ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi, environmentVariableProvider); } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 3bc5ee589..80f842024 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -11,7 +11,7 @@ public class GenerateScriptCommandArgs : CommandArgs public string GithubOrg { get; set; } public string GitlabUsername { get; set; } [Secret] - public string GitlabPassword { get; set; } + public string GitlabPat { get; set; } public string GitlabGroup { get; set; } public string GitlabSharedHome { get; set; } public string ArchiveDownloadHost { get; set; } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 93dcbedb9..79632fbfa 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -59,7 +59,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(VALIDATE_GH_PAT); if (!args.Kerberos) { - content.AppendLine(VALIDATE_BBS_PASSWORD); + content.AppendLine(VALIDATE_GITLAB_PAT); } if (args.GitlabUsername.IsNullOrWhiteSpace() && !args.Kerberos) { @@ -175,12 +175,12 @@ exit 1 } else { Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" }"; - private const string VALIDATE_BBS_PASSWORD = @" -if (-not $env:BBS_PASSWORD) { - Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + private const string VALIDATE_GITLAB_PAT = @" +if (-not $env:GITLAB_PAT) { + Write-Error ""GITLAB_PAT environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" exit 1 } else { - Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" + Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" }"; private const string VALIDATE_AZURE_STORAGE_CONNECTION_STRING = @" if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 7b90535e9..d715ff3e2 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -17,7 +17,7 @@ public InventoryReportCommand() : base( { AddOption(GitlabServerUrl); AddOption(GitlabGroup); - AddOption(GitlabPassword); + AddOption(GitlabPat); AddOption(NoSslVerify); AddOption(Minimal); AddOption(Verbose); @@ -25,15 +25,15 @@ public InventoryReportCommand() : base( public Option GitlabServerUrl { get; } = new( name: "--gitlab-server-url", - description: "The full URL of the Bitbucket Server/Data Center. E.g. http://bitbucket.contoso.com:7990") + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com") { IsRequired = true }; public Option GitlabGroup { get; } = new( name: "--gitlab-group", description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); - public Option GitlabPassword { get; } = new( - name: "--gitlab-password", + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", description: "The Bitbucket password of the user specified by --gitlab-username. If not set will be read from BBS_PASSWORD environment variable."); public Option NoSslVerify { get; } = new( @@ -61,7 +61,7 @@ public override InventoryReportCommandHandler BuildHandler(InventoryReportComman var log = sp.GetRequiredService(); var gitlabApiFactory = sp.GetRequiredService(); - var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); var gitlabInspectorServiceFactory = sp.GetRequiredService(); var gitlabInspectorService = gitlabInspectorServiceFactory.Create(gitlabApi); var groupsCsvGeneratorService = sp.GetRequiredService(); diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs index 129b52435..eb6f40e95 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -8,7 +8,7 @@ public class InventoryReportCommandArgs : CommandArgs public string GitlabGroup { get; set; } public string GitlabUsername { get; set; } [Secret] - public string GitlabPassword { get; set; } + public string GitlabPat { get; set; } public bool NoSslVerify { get; set; } public bool Minimal { get; set; } } diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 72bf03327..a4eebed8c 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -54,12 +54,12 @@ public async Task Handle(InventoryReportCommandArgs args) _log.LogInformation($"Found {projectCount} Projects"); _log.LogInformation("Generating data for groups.csv..."); - var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); + var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); await WriteToFile("groups.csv", groupsCsvText); _log.LogSuccess("groups.csv generated"); _log.LogInformation("Generating projects.csv..."); - var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify, args.GitlabGroup, args.Minimal); + var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); await WriteToFile("projects.csv", projectsCsvText); _log.LogSuccess("projects.csv generated"); } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 748879079..28be1e8ad 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -15,7 +15,7 @@ public class MigrateRepoCommand : CommandBase GitlabPassword { get; } = new( - name: "--bbs-password", - description: "The Bitbucket password of the user specified by --bbs-username. If not set will be read from BBS_PASSWORD environment variable."); + public Option GitlabPat { get; } = new( + name: "--bbs-pat", + description: "The Bitbucket PAT of the user specified by --bbs-username. If not set will be read from GITLAB_PAT environment variable."); public Option GitlabSharedHome { get; } = new( name: "--bbs-shared-home", @@ -233,7 +233,7 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar gitlabApi = args.Kerberos ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPassword, args.NoSslVerify); + : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); } if (args.SshUser.HasValue() || args.SmbUser.HasValue()) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index 3ad44a349..ffc768c94 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -36,7 +36,7 @@ public class MigrateRepoCommandArgs : CommandArgs public string GitlabGroup { get; set; } public string GitlabProject { get; set; } [Secret] - public string GitlabPassword { get; set; } + public string GitlabPat { get; set; } public string GitlabSharedHome { get; set; } public bool NoSslVerify { get; set; } @@ -93,9 +93,9 @@ public override void Validate(OctoLogger log) private void ValidateNoGenerateOptions() { - if (GitlabPassword.HasValue()) + if (GitlabPat.HasValue()) { - throw new OctoshiftCliException("--gitlab-password cannot be provided with --archive-path or --archive-url."); + throw new OctoshiftCliException("--gitlab-pat cannot be provided with --archive-path or --archive-url."); } if (NoSslVerify) @@ -120,9 +120,9 @@ private void ValidateNoGenerateOptions() private void ValidateGenerateOptions() { - if (Kerberos && GitlabPassword.HasValue()) + if (Kerberos && GitlabPat.HasValue()) { - throw new OctoshiftCliException("--gitlab-password cannot be provided with --kerberos."); + throw new OctoshiftCliException("--gitlab-pat cannot be provided with --kerberos."); } if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index 26223526b..4a8a2957d 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -307,7 +307,7 @@ private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => a ? args.AzureStorageConnectionString : _environmentVariableProvider.AzureStorageConnectionString(false); - private string GetGitlabPassword(MigrateRepoCommandArgs args) => args.GitlabPassword.HasValue() ? args.GitlabPassword : _environmentVariableProvider.GitlabPassword(false); + private string GetGitlabPat(MigrateRepoCommandArgs args) => args.GitlabPat.HasValue() ? args.GitlabPat : _environmentVariableProvider.GitlabPat(false); private string GetSmbPassword(MigrateRepoCommandArgs args) => args.SmbPassword.HasValue() ? args.SmbPassword : _environmentVariableProvider.SmbPassword(false); @@ -324,9 +324,9 @@ private void ValidateOptions(MigrateRepoCommandArgs args) { if (!args.Kerberos) { - if (GetGitlabPassword(args).IsNullOrWhiteSpace()) + if (GetGitlabPat(args).IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("BBS password must be either set as BBS_PASSWORD environment variable or passed as --bbs-password."); + throw new OctoshiftCliException("BBS password must be either set as BBS_PAT environment variable or passed as --bbs-pat."); } } diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs index 55dd47046..ae4dad73b 100644 --- a/src/gl2gh/Factories/GitlabApiFactory.cs +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -21,14 +21,14 @@ public GitlabApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory, _retryPolicy = retryPolicy; } - public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPassword, bool noSsl = false) + public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool noSsl = false) { - gitlabPassword ??= _environmentVariableProvider.GitlabPassword(); + gitlabPat ??= _environmentVariableProvider.GitlabPat(); var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); - var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPassword); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat); return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs index be38c9372..0a473dac2 100644 --- a/src/gl2gh/Services/GroupsCsvGeneratorService.cs +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -16,11 +16,11 @@ public GroupsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorSe _gitlabApiFactory = gitlabApiFactory; } - public virtual async Task Generate(string gitlabServerUrl, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) { gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); - var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPassword, noSslVerify); + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index b84562b8a..59ba61fe2 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -16,11 +16,11 @@ public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspector _gitlabApiFactory = gitlabApiFactory; } - public virtual async Task Generate(string gitlabServerUrl, string gitlabPassword, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) { gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); - var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPassword, noSslVerify); + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); From 243d3afd3ee5e17f6ed5b4c5551180599e3f2470 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 12:25:39 -0700 Subject: [PATCH 43/71] Use IGitlabArchiveDownloader.cs for IGitlabArchiveDownloader. --- .../{IBbsArchiveDownloader.cs => IGitlabArchiveDownloader.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/gl2gh/Services/{IBbsArchiveDownloader.cs => IGitlabArchiveDownloader.cs} (100%) diff --git a/src/gl2gh/Services/IBbsArchiveDownloader.cs b/src/gl2gh/Services/IGitlabArchiveDownloader.cs similarity index 100% rename from src/gl2gh/Services/IBbsArchiveDownloader.cs rename to src/gl2gh/Services/IGitlabArchiveDownloader.cs From 23e9fc544566961a439e5ab658504128f2702ac5 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 14:10:32 -0700 Subject: [PATCH 44/71] Remove Samba and SSH options from GL GenerateScriptCommandHandler. --- .../GenerateScriptCommandHandler.cs | 34 +-- .../Services/GitlabSmbArchiveDownloader.cs | 229 ------------------ .../Services/GitlabSshArchiveDownloader.cs | 127 ---------- 3 files changed, 3 insertions(+), 387 deletions(-) delete mode 100644 src/gl2gh/Services/GitlabSmbArchiveDownloader.cs delete mode 100644 src/gl2gh/Services/GitlabSshArchiveDownloader.cs diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 79632fbfa..f0a6e71fb 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -61,10 +61,6 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) { content.AppendLine(VALIDATE_GITLAB_PAT); } - if (args.GitlabUsername.IsNullOrWhiteSpace() && !args.Kerberos) - { - content.AppendLine(VALIDATE_BBS_USERNAME); - } if (args.AwsBucketName.HasValue() || args.AwsRegion.HasValue()) { content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); @@ -74,10 +70,6 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) { content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); } - if (args.SmbUser.HasValue()) - { - content.AppendLine(VALIDATE_SMB_PASSWORD); - } var groups = args.GitlabGroup.HasValue() ? [args.GitlabGroup] @@ -122,12 +114,6 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb var waitOption = wait ? "" : " --queue-only"; var kerberosOption = args.Kerberos ? " --kerberos" : ""; var verboseOption = args.Verbose ? " --verbose" : ""; - var sshArchiveDownloadOptions = args.SshUser.HasValue() - ? $" --ssh-user \"{args.SshUser}\" --ssh-private-key \"{args.SshPrivateKey}\"{(args.SshPort.HasValue() ? $" --ssh-port {args.SshPort}" : "")}{(args.ArchiveDownloadHost.HasValue() ? $" --archive-download-host {args.ArchiveDownloadHost}" : "")}" : ""; - var smbArchiveDownloadOptions = args.SmbUser.HasValue() - ? $" --smb-user \"{args.SmbUser}\"{(args.SmbDomain.HasValue() ? $" --smb-domain {args.SmbDomain}" : "")}{(args.ArchiveDownloadHost.HasValue() ? $" --archive-download-host {args.ArchiveDownloadHost}" : "")}" - : ""; - var bbsSharedHomeOption = args.GitlabSharedHome.HasValue() ? $" --bbs-shared-home \"{args.GitlabSharedHome}\"" : ""; var awsBucketNameOption = args.AwsBucketName.HasValue() ? $" --aws-bucket-name \"{args.AwsBucketName}\"" : ""; var awsRegionOption = args.AwsRegion.HasValue() ? $" --aws-region \"{args.AwsRegion}\"" : ""; var keepArchive = args.KeepArchive ? " --keep-archive" : ""; @@ -137,8 +123,8 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; - return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsSharedHomeOption}{bbsProjectOption}{bbsRepoOption}{sshArchiveDownloadOptions}" + - $"{smbArchiveDownloadOptions}{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; + return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsProjectOption}{bbsRepoOption}" + + $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; } private string Exec(string script) => Wrap(script, "Exec"); @@ -167,17 +153,10 @@ function Exec { exit 1 } else { Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" -}"; - private const string VALIDATE_BBS_USERNAME = @" -if (-not $env:BBS_USERNAME) { - Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" - exit 1 -} else { - Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" }"; private const string VALIDATE_GITLAB_PAT = @" if (-not $env:GITLAB_PAT) { - Write-Error ""GITLAB_PAT environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" + Write-Error ""GITLAB_PAT environment variable must be set to a valid PAT that will be used to call the GitLab API to generate a migration archive."" exit 1 } else { Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" @@ -202,12 +181,5 @@ exit 1 exit 1 } else { Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" -}"; - private const string VALIDATE_SMB_PASSWORD = @" -if (-not $env:SMB_PASSWORD) { - Write-Error ""SMB_PASSWORD environment variable must be set to a valid password that will be used to download the migration archive from your BBS server using SMB."" - exit 1 -} else { - Write-Host ""SMB_PASSWORD environment variable is set and will be used to download the migration archive from your BBS server using SMB."" }"; } diff --git a/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs b/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs deleted file mode 100644 index 08ff6d6e7..000000000 --- a/src/gl2gh/Services/GitlabSmbArchiveDownloader.cs +++ /dev/null @@ -1,229 +0,0 @@ -using System; -using System.IO; -using System.Net; -using System.Threading.Tasks; -using OctoshiftCLI.Extensions; -using OctoshiftCLI.Services; -using SMBLibrary; -using SMBLibrary.Client; -using FileAttributes = SMBLibrary.FileAttributes; - -namespace OctoshiftCLI.GitlabToGithub.Services; - -public sealed class GitlabSmbArchiveDownloader : IGitlabArchiveDownloader -{ - private const int DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; - - private readonly ISMBClient _smbClient; - private readonly OctoLogger _log; - private readonly FileSystemProvider _fileSystemProvider; - private readonly string _host; - private readonly string _smbUser; - private readonly string _smbPassword; - private readonly string _domainName; - private DateTime _nextProgressReport; - - public GitlabSmbArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, string host, string smbUser, string smbPassword, string domainName = null) - : this(log, fileSystemProvider, new SMB2Client(), host, smbUser, smbPassword, domainName) - { - } - - internal GitlabSmbArchiveDownloader( - OctoLogger log, - FileSystemProvider fileSystemProvider, - ISMBClient smbClient, - string host, - string smbUser, - string smbPassword, - string domainName = null) - { - _log = log ?? throw new ArgumentNullException(nameof(log)); - _fileSystemProvider = fileSystemProvider ?? throw new ArgumentNullException(nameof(fileSystemProvider)); - _smbClient = smbClient ?? throw new ArgumentNullException(nameof(smbClient)); - _host = host ?? throw new ArgumentNullException(nameof(host)); - _smbUser = smbUser; - _smbPassword = smbPassword; - _domainName = domainName; - } - - public string GitlabSharedHomeDirectory { get; init; } = GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS; - - private string GetSourceExportArchiveAbsolutePath(long exportJobId) => - IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(GitlabSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS, exportJobId).ToWindowsPath(); - - public async Task Download(long exportJobId, string targetDirectory = IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY) - { - _nextProgressReport = DateTime.Now; - - ISMBFileStore fileStore = null; - object sourceExportArchiveHandle = null; - - var sourceExportArchiveFullPath = GetSourceExportArchiveAbsolutePath(exportJobId); - var share = sourceExportArchiveFullPath[..sourceExportArchiveFullPath.IndexOf("\\", StringComparison.Ordinal)]; - var sourceExportArchivePathAfterShare = sourceExportArchiveFullPath[(sourceExportArchiveFullPath.IndexOf("\\", StringComparison.Ordinal) + 1)..]; - - var targetExportArchiveFullPath = - Path.Join(targetDirectory ?? IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY, IGitlabArchiveDownloader.GetExportArchiveFileName(exportJobId)).ToUnixPath(); - - await using var targetExportArchive = OpenWriteTargetExportArchive(targetExportArchiveFullPath); - - try - { - ConnectToHost(); - Login(); - - fileStore = CreateSmbFileStore(share); - sourceExportArchiveHandle = CreateFileHandle(fileStore, sourceExportArchivePathAfterShare); - var sourceExportArchiveSize = GetFileSize(fileStore, sourceExportArchiveHandle); - - long bytesRead = 0; - while (true) - { - var status = fileStore.ReadFile(out var data, sourceExportArchiveHandle, bytesRead, (int)_smbClient.MaxReadSize); - - if (IsEndOfFileStatus(status) || data.Length == 0) - { - break; - } - - if (!IsSuccessStatus(status)) - { - throw new OctoshiftCliException($"Failed to read from source export archive \"{sourceExportArchiveFullPath}\" (Status Code: {status})."); - } - - bytesRead += data.Length; - await _fileSystemProvider.WriteAsync(targetExportArchive, data); - - LogProgress(bytesRead, sourceExportArchiveSize); - } - - return targetExportArchiveFullPath; - } - finally - { - if (sourceExportArchiveHandle != null) - { - fileStore?.CloseFile(sourceExportArchiveHandle); - } - - fileStore?.Disconnect(); - _smbClient.Logoff(); - _smbClient.Disconnect(); - } - } - - private void LogProgress(long downloadedBytes, long? totalBytes) - { - if (DateTime.Now < _nextProgressReport) - { - return; - } - - var totalProgressMessage = totalBytes.HasValue - ? $" out of {GetLogFriendlySize(totalBytes.Value)} ({GetPercentage(downloadedBytes, totalBytes.Value)})" - : ""; - _log.LogInformation($"Archive download in progress, {GetLogFriendlySize(downloadedBytes)}{totalProgressMessage} completed..."); - - _nextProgressReport = _nextProgressReport.AddSeconds(DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); - } - - private string GetPercentage(long downloadedBytes, long totalBytes) - { - if (totalBytes is 0L) - { - return "unknown%"; - } - - var percentage = (int)(downloadedBytes * 100D / totalBytes); - return $"{percentage}%"; - } - - private string GetLogFriendlySize(long size) - { - const int kilobyte = 1024; - const int megabyte = 1024 * kilobyte; - const int gigabyte = 1024 * megabyte; - - return size switch - { - < kilobyte => $"{size:n0} bytes", - < megabyte => $"{size / (double)kilobyte:n0} KB", - < gigabyte => $"{size / (double)megabyte:n0} MB", - _ => $"{size / (double)gigabyte:n2} GB" - }; - } - - private FileStream OpenWriteTargetExportArchive(string targetExportArchiveFullPath) - { - _fileSystemProvider.CreateDirectory(Path.GetDirectoryName(targetExportArchiveFullPath)); - return _fileSystemProvider.Open(targetExportArchiveFullPath, FileMode.Create); - } - - private void ConnectToHost() - { - var isConnected = IPAddress.TryParse(_host, out var ipAddress) - ? _smbClient.Connect(ipAddress, SMBTransportType.DirectTCPTransport) - : _smbClient.Connect(_host, SMBTransportType.DirectTCPTransport); - - if (!isConnected) - { - throw new OctoshiftCliException($"Unable to connect to host \"{_host}\"."); - } - } - - private void Login() - { - var status = _smbClient.Login(_domainName ?? "", _smbUser, _smbPassword); - - if (!IsSuccessStatus(status)) - { - throw new OctoshiftCliException($"Unable to login with provided credentials (Status Code: {status})."); - } - } - - private ISMBFileStore CreateSmbFileStore(string shareName) - { - var fileStore = _smbClient.TreeConnect(shareName, out var status); - - return IsSuccessStatus(status) - ? fileStore - : throw new OctoshiftCliException($"Unable to connect to share \"{shareName}\" (Status Code: {status}). " + - "Please make sure that the directory is shared and the share name is correct."); - } - - private object CreateFileHandle(ISMBFileStore fileStore, string sharedFilePath) - { - var status = fileStore.CreateFile( - out var sharedFileHandle, - out _, - sharedFilePath, - AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.Read, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null); - - return IsSuccessStatus(status) - ? sharedFileHandle - : throw new OctoshiftCliException( - $"Couldn't create SMB file handle for \"{sharedFilePath}\" (Status Code: {status})." + - (IsObjectPathNotFoundStatus(status) && GitlabSharedHomeDirectory is GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS - ? "This most likely means that your Bitbucket instance uses a non-default Bitbucket shared home directory, so we couldn't find your archive. " + - "You can point the CLI to a non-default shared directory by specifying the --bbs-shared-home option." - : "")); - } - - private long? GetFileSize(ISMBFileStore fileStore, object sharedFileHandle) - { - var status = fileStore.GetFileInformation(out var fileInfo, sharedFileHandle, FileInformationClass.FileStandardInformation); - - return !IsSuccessStatus(status) ? null : (fileInfo as FileStandardInformation)?.AllocationSize; - } - - private bool IsSuccessStatus(NTStatus status) => status is NTStatus.STATUS_SUCCESS; - - private bool IsEndOfFileStatus(NTStatus status) => status is NTStatus.STATUS_END_OF_FILE; - - private bool IsObjectPathNotFoundStatus(NTStatus status) => status is NTStatus.STATUS_OBJECT_PATH_NOT_FOUND; -} diff --git a/src/gl2gh/Services/GitlabSshArchiveDownloader.cs b/src/gl2gh/Services/GitlabSshArchiveDownloader.cs deleted file mode 100644 index a135227d9..000000000 --- a/src/gl2gh/Services/GitlabSshArchiveDownloader.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using OctoshiftCLI.Extensions; -using OctoshiftCLI.Services; -using Renci.SshNet; - -namespace OctoshiftCLI.GitlabToGithub.Services; - -public sealed class GitlabSshArchiveDownloader : IGitlabArchiveDownloader, IDisposable -{ - private const int DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS = 10; - - private readonly ISftpClient _sftpClient; - private readonly PrivateKeyFile _privateKey; - private readonly OctoLogger _log; - private readonly FileSystemProvider _fileSystemProvider; - private readonly object _mutex = new(); - private DateTime _nextProgressReport; - - public GitlabSshArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, string host, string sshUser, string privateKeyFileFullPath, int sshPort = 22) - { - _log = log; - _fileSystemProvider = fileSystemProvider; - _privateKey = new PrivateKeyFile(privateKeyFileFullPath); - _sftpClient = new SftpClient(host, sshPort, sshUser, _privateKey); - } - - internal GitlabSshArchiveDownloader(OctoLogger log, FileSystemProvider fileSystemProvider, ISftpClient sftpClient) - { - _log = log; - _fileSystemProvider = fileSystemProvider; - _sftpClient = sftpClient; - } - - public string GitlabSharedHomeDirectory { get; init; } = GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX; - - private string GetSourceExportArchiveAbsolutePath(long exportJobId) => - IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(GitlabSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX, exportJobId).ToUnixPath(); - - public async Task Download(long exportJobId, string targetDirectory = IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY) - { - _nextProgressReport = DateTime.Now; - - var sourceExportArchiveFullPath = GetSourceExportArchiveAbsolutePath(exportJobId); - var targetExportArchiveFullPath = - Path.Join(targetDirectory ?? IGitlabArchiveDownloader.DEFAULT_TARGET_DIRECTORY, IGitlabArchiveDownloader.GetExportArchiveFileName(exportJobId)).ToUnixPath(); - - if (_sftpClient is BaseClient { IsConnected: false } client) - { - client.Connect(); - } - - if (!_sftpClient.Exists(sourceExportArchiveFullPath)) - { - throw new OctoshiftCliException( - $"Source export archive ({sourceExportArchiveFullPath}) does not exist." + - (GitlabSharedHomeDirectory is GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX - ? "This most likely means that your Bitbucket instance uses a non-default Bitbucket shared home directory, so we couldn't find your archive. " + - "You can point the CLI to a non-default shared directory by specifying the --bbs-shared-home option." - : "")); - } - - _fileSystemProvider.CreateDirectory(targetDirectory); - - var sourceExportArchiveSize = _sftpClient.GetAttributes(sourceExportArchiveFullPath)?.Size ?? long.MaxValue; - await using var targetExportArchive = _fileSystemProvider.Open(targetExportArchiveFullPath, FileMode.Create); - await Task.Factory.FromAsync( - _sftpClient.BeginDownloadFile( - sourceExportArchiveFullPath, - targetExportArchive, - null, - null, - downloaded => LogProgress(downloaded, (ulong)sourceExportArchiveSize)), - _sftpClient.EndDownloadFile); - - return targetExportArchiveFullPath; - } - - private void LogProgress(ulong downloadedBytes, ulong totalBytes) - { - lock (_mutex) - { - if (DateTime.Now < _nextProgressReport) - { - return; - } - - _log.LogInformation( - $"Archive download in progress, {GetLogFriendlySize(downloadedBytes)} out of {GetLogFriendlySize(totalBytes)} ({GetPercentage(downloadedBytes, totalBytes)}) completed..."); - - _nextProgressReport = _nextProgressReport.AddSeconds(DOWNLOAD_PROGRESS_REPORT_INTERVAL_IN_SECONDS); - } - } - - private string GetPercentage(ulong downloadedBytes, ulong totalBytes) - { - if (totalBytes is ulong.MinValue) - { - return "unknown%"; - } - - var percentage = (int)(downloadedBytes * 100D / totalBytes); - return $"{percentage}%"; - } - - private string GetLogFriendlySize(ulong size) - { - const int kilobyte = 1024; - const int megabyte = 1024 * kilobyte; - const int gigabyte = 1024 * megabyte; - - return size switch - { - < kilobyte => $"{size:n0} bytes", - < megabyte => $"{size / (double)kilobyte:n0} KB", - < gigabyte => $"{size / (double)megabyte:n0} MB", - _ => $"{size / (double)gigabyte:n2} GB" - }; - } - - public void Dispose() - { - _sftpClient?.Dispose(); - _privateKey?.Dispose(); - } -} From 3334f136bc5bb9e6d741e7a6f9ebb16096a4ea7f Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 14:22:39 -0700 Subject: [PATCH 45/71] Update GL GenerateScriptCommandHandler to use GitLab options. --- .../GenerateScriptCommandHandler.cs | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index f0a6e71fb..2c60d8881 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -82,35 +82,34 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(); content.AppendLine($"# =========== Group: {groupPath} ==========="); - var repos = await _gitlabApi.GetProjects(groupPath); + var projects = await _gitlabApi.GetProjects(groupPath); - if (!repos.Any()) + if (!projects.Any()) { - content.AppendLine("# Skipping this group because it has no git repos."); + content.AppendLine("# Skipping this group because it has no projects."); continue; } content.AppendLine(); - foreach (var (_, repoSlug, repoName, _) in repos) + foreach (var (_, projectPath, projectName, _) in projects) { - _log.LogInformation($" Repo: {repoName}"); + _log.LogInformation($" Project: {projectName}"); - content.AppendLine(Exec(MigrateGithubRepoScript(args, groupPath, repoSlug, true))); + content.AppendLine(Exec(MigrateGithubRepoScript(args, groupPath, projectPath, true))); } } return content.ToString(); } - private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bbsProjectKey, string bbsRepoSlug, bool wait) + private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string gitlabGroup, string gitlabProject, bool wait) { - var bbsServerUrlOption = $" --bbs-server-url \"{args.GitlabServerUrl}\""; - var bbsUsernameOption = args.GitlabUsername.HasValue() ? $" --bbs-username \"{args.GitlabUsername}\"" : ""; - var bbsProjectOption = $" --bbs-group \"{bbsProjectKey}\""; - var bbsRepoOption = $" --bbs-repo \"{bbsRepoSlug}\""; + var gitlabServerUrlOption = $" --gitlab-server-url \"{args.GitlabServerUrl}\""; + var gitlabGroupOption = $" --gitlab-group \"{gitlabGroup}\""; + var gitlabProjectOption = $" --gitlab-project \"{gitlabProject}\""; var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; - var githubRepoOption = $" --github-repo \"{GetGithubRepoName(bbsProjectKey, bbsRepoSlug)}\""; + var githubRepoOption = $" --github-repo \"{GetGithubRepoName(gitlabGroup, gitlabProject)}\""; var waitOption = wait ? "" : " --queue-only"; var kerberosOption = args.Kerberos ? " --kerberos" : ""; var verboseOption = args.Verbose ? " --verbose" : ""; @@ -123,7 +122,7 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; - return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{bbsServerUrlOption}{bbsUsernameOption}{bbsProjectOption}{bbsRepoOption}" + + return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{gitlabServerUrlOption}{gitlabGroupOption}{gitlabProjectOption}" + $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; } @@ -131,7 +130,7 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string bb private string Wrap(string script, string outerCommand = "") => script.IsNullOrWhiteSpace() ? string.Empty : $"{outerCommand} {{ {script} }}".Trim(); - private string GetGithubRepoName(string bbsProjectKey, string bbsRepoSlug) => $"{bbsProjectKey}-{bbsRepoSlug}".ReplaceInvalidCharactersWithDash(); + private string GetGithubRepoName(string gitlabGroup, string gitlabProject) => $"{gitlabGroup}-{gitlabProject}".ReplaceInvalidCharactersWithDash(); private string VersionComment => $"# =========== Created with CLI version {_versionProvider.GetCurrentVersion()} ==========="; @@ -159,7 +158,7 @@ exit 1 Write-Error ""GITLAB_PAT environment variable must be set to a valid PAT that will be used to call the GitLab API to generate a migration archive."" exit 1 } else { - Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" + Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to the GitLab API."" }"; private const string VALIDATE_AZURE_STORAGE_CONNECTION_STRING = @" if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { From 72ac8ec09f870ab3be082551c28640cea9c9ad55 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 14:32:28 -0700 Subject: [PATCH 46/71] Remove Message from GotlabApi GetExport response. --- src/Octoshift/Services/GitlabApi.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index b2ee4ea04..b39f70f5b 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -40,7 +40,7 @@ public virtual async Task StartExport(string groupPath, string projectPath return (long)exportData["id"]; } - public virtual async Task<(string ExportStatus, string Message, string DownloadUrl)> GetExport(string groupPath, string projectPath) + public virtual async Task<(string ExportStatus, string DownloadUrl)> GetExport(string groupPath, string projectPath) { var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; @@ -50,7 +50,6 @@ public virtual async Task StartExport(string groupPath, string projectPath return ( (string)exportData["export_status"], - (string)exportData["message"], (string)exportData["_links"]?["api_url"] ); } From 1fab75a2356cdbc1a7a6a71de4ccb4229091bfcf Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 14:42:38 -0700 Subject: [PATCH 47/71] Update GL MigrateRepoCommand to use GitLab options. --- .../MigrateRepo/MigrateRepoCommand.cs | 103 ++++-------------- 1 file changed, 22 insertions(+), 81 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 28be1e8ad..23ebe5fdf 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -24,17 +24,9 @@ public MigrateRepoCommand() : base( AddOption(GithubRepo); AddOption(GithubPat); AddOption(GitlabServerUrl); + AddOption(GitlabGroup); AddOption(GitlabProject); - AddOption(GitlabRepo); AddOption(GitlabPat); - AddOption(GitlabSharedHome); - AddOption(SshUser); - AddOption(SshPrivateKey); - AddOption(SshPort); - AddOption(ArchiveDownloadHost); - AddOption(SmbUser); - AddOption(SmbPassword); - AddOption(SmbDomain); AddOption(ArchivePath); AddOption(AzureStorageConnectionString); AddOption(AwsBucketName); @@ -54,51 +46,46 @@ public MigrateRepoCommand() : base( } public Option GitlabServerUrl { get; } = new( - name: "--bbs-server-url", - description: "The full URL of the Bitbucket Server/Data Center to migrate from. E.g. http://bitbucket.contoso.com:7990") + name: "--gitlab-server-url", + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com") { IsRequired = true }; - public Option GitlabProject { get; } = new( - name: "--bbs-project", - description: "The Bitbucket project to migrate.") + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group to migrate.") { IsRequired = true }; - public Option GitlabRepo { get; } = new( - name: "--bbs-repo", - description: "The Bitbucket repository to migrate.") + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate.") { IsRequired = true }; public Option GitlabPat { get; } = new( - name: "--bbs-pat", - description: "The Bitbucket PAT of the user specified by --bbs-username. If not set will be read from GITLAB_PAT environment variable."); - - public Option GitlabSharedHome { get; } = new( - name: "--bbs-shared-home", - description: "Bitbucket server's shared home directory. Defaults to \"/var/atlassian/application-data/bitbucket/shared\" if downloading the archive from a server using SSH " + - "and \"c$\\atlassian\\applicationdata\\bitbucket\\shared\" if downloading using SMB."); + name: "--gitlab-pat", + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); public Option ArchiveUrl { get; } = new( name: "--archive-url", description: - "URL used to download Bitbucket Server migration archive. Only needed if you want to manually retrieve the archive from BBS instead of letting this CLI do that for you."); + "URL used to download the GitLab migration archive. Only needed if you want to manually retrieve the archive from GitLab instead of letting this CLI do that for you."); public Option ArchivePath { get; } = new( name: "--archive-path", - description: "Path to Bitbucket Server migration archive on disk."); + description: "Path to the GitLab migration archive on disk."); public Option AzureStorageConnectionString { get; } = new( name: "--azure-storage-connection-string", - description: "A connection string for an Azure Storage account, used to upload the BBS archive. If not set will be read from AZURE_STORAGE_CONNECTION_STRING environment variable."); + description: "A connection string for an Azure Storage account, used to upload the GitLab archive. If not passed, it will read the AZURE_STORAGE_CONNECTION_STRING environment variable."); public Option AwsBucketName { get; } = new( name: "--aws-bucket-name", - description: "If using AWS, the name of the S3 bucket to upload the BBS archive to."); + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); public Option AwsAccessKey { get; } = new( name: "--aws-access-key", @@ -121,45 +108,6 @@ public MigrateRepoCommand() : base( public Option GithubRepo { get; } = new("--github-repo"); - public Option ArchiveDownloadHost { get; } = new( - name: "--archive-download-host", - description: "The host to use to connect to the Bitbucket Server/Data Center instance via SSH or SMB. Defaults to the host from the Bitbucket Server URL (--bbs-server-url)."); - - public Option SshUser { get; } = new( - name: "--ssh-user", - description: "The SSH user to be used for downloading the export archive off of the Bitbucket server."); - - public Option SshPrivateKey { get; } = new( - name: "--ssh-private-key", - description: "The full path of the private key file to be used for downloading the export archive off of the Bitbucket Server using SSH/SFTP." + - Environment.NewLine + - "Supported private key formats:" + - Environment.NewLine + - " - RSA in OpenSSL PEM format." + - Environment.NewLine + - " - DSA in OpenSSL PEM format." + - Environment.NewLine + - " - ECDSA 256/384/521 in OpenSSL PEM format." + - Environment.NewLine + - " - ECDSA 256/384/521, ED25519 and RSA in OpenSSH key format."); - - public Option SshPort { get; } = new( - name: "--ssh-port", - getDefaultValue: () => 22, - description: "The SSH port (default: 22)."); - - public Option SmbUser { get; } = new( - name: "--smb-user", - description: "The SMB user used for authentication when downloading the export archive from the Bitbucket Server instance."); - - public Option SmbPassword { get; } = new( - name: "--smb-password", - description: "The SMB password used for authentication when downloading the export archive from the Bitbucket server instance. If not provided, it will be read from SMB_PASSWORD environment variable."); - - public Option SmbDomain { get; } = new( - name: "--smb-domain", - description: "The optional domain name when using SMB for downloading the export archive."); - public Option GithubPat { get; } = new( name: "--github-pat", description: "The GitHub personal access token to be used for the migration. If not set will be read from GH_PAT environment variable."); @@ -174,7 +122,7 @@ public MigrateRepoCommand() : base( public Option Kerberos { get; } = new( name: "--kerberos", - description: "Use Kerberos authentication for downloading the export archive off of the Bitbucket server.") + description: "Use Kerberos authentication for downloading the export archive off of the GitLab server.") { IsHidden = true }; public Option Verbose { get; } = new("--verbose"); @@ -192,8 +140,8 @@ public MigrateRepoCommand() : base( { IsHidden = true }; public Option NoSslVerify { get; } = new( name: "--no-ssl-verify", - description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. All other migration steps will continue to verify SSL. " + - "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow the migration archive to be exported."); + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); public Option UseGithubStorage { get; } = new( name: "--use-github-storage", description: "Enables multipart uploads to a GitHub owned storage for use during migration. " + @@ -217,7 +165,7 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar var fileSystemProvider = sp.GetRequiredService(); GithubApi githubApi = null; GitlabApi gitlabApi = null; - IGitlabArchiveDownloader bbsArchiveDownloader = null; + IGitlabArchiveDownloader gitlabArchiveDownloader = null; AzureApi azureApi = null; AwsApi awsApi = null; @@ -236,15 +184,8 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); } - if (args.SshUser.HasValue() || args.SmbUser.HasValue()) - { - var bbsArchiveDownloaderFactory = sp.GetRequiredService(); - var bbsHost = args.ArchiveDownloadHost.HasValue() ? args.ArchiveDownloadHost : new Uri(args.GitlabServerUrl).Host; - - bbsArchiveDownloader = args.SshUser.HasValue() - ? bbsArchiveDownloaderFactory.CreateSshDownloader(bbsHost, args.SshUser, args.SshPrivateKey, args.SshPort, args.GitlabSharedHome) - : bbsArchiveDownloaderFactory.CreateSmbDownloader(bbsHost, args.SmbUser, args.SmbPassword, args.SmbDomain, args.GitlabSharedHome); - } + var gitlabArchiveDownloaderFactory = sp.GetRequiredService(); + gitlabArchiveDownloader = gitlabArchiveDownloaderFactory.CreateDownloader(); var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); if (azureStorageConnectionString.HasValue()) @@ -261,6 +202,6 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar var warningsCountLogger = sp.GetRequiredService(); - return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, bbsArchiveDownloader, azureApi, awsApi, fileSystemProvider, warningsCountLogger); + return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, gitlabArchiveDownloader, azureApi, awsApi, fileSystemProvider, warningsCountLogger); } } From e2178a948147e00448f4ceeffd51fdb2571115be Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 16:35:52 -0700 Subject: [PATCH 48/71] Use GitLab states in GL ExportState. --- src/gl2gh/ExportState.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/gl2gh/ExportState.cs b/src/gl2gh/ExportState.cs index a406304e2..c403e3b56 100644 --- a/src/gl2gh/ExportState.cs +++ b/src/gl2gh/ExportState.cs @@ -2,11 +2,10 @@ namespace OctoshiftCLI.GitlabToGithub; public static class ExportState { - public const string COMPLETED = "COMPLETED"; - public const string FAILED = "FAILED"; - public const string ABORTED = "ABORTED"; + public const string FINISHED = "finished"; + public const string FAILED = "failed"; - public static bool IsInProgress(string state) => state is not COMPLETED && !IsError(state); + public static bool IsInProgress(string state) => state is not FINISHED && !IsError(state); - public static bool IsError(string state) => state is FAILED or ABORTED; + public static bool IsError(string state) => state is FAILED; } From 662dd87a14373791af85b519908fa178d491bb50 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 16:36:22 -0700 Subject: [PATCH 49/71] Return message from GitlabApi StartExport. --- src/Octoshift/Services/GitlabApi.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index b39f70f5b..5726c35e3 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -29,7 +29,7 @@ public virtual async Task GetServerVersion() return (string)JObject.Parse(content)["version"]; } - public virtual async Task StartExport(string groupPath, string projectPath) + public virtual async Task StartExport(string groupPath, string projectPath) { var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; @@ -37,7 +37,7 @@ public virtual async Task StartExport(string groupPath, string projectPath var exportResponse = await _client.PostAsync(url, new { }); var exportData = JObject.Parse(exportResponse); - return (long)exportData["id"]; + return (string)exportData["message"]; } public virtual async Task<(string ExportStatus, string DownloadUrl)> GetExport(string groupPath, string projectPath) From 3892154ba56fe31d1c0d3328f33c56ad4f6168cf Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:42:59 -0700 Subject: [PATCH 50/71] Remove GitLab archive downloader. --- .../MigrateRepo/MigrateRepoCommand.cs | 1 - .../MigrateRepo/MigrateRepoCommandHandler.cs | 1 - .../GitlabArchiveDownloaderFactory.cs | 30 ------------------- src/gl2gh/Program.cs | 1 - .../Services/IGitlabArchiveDownloader.cs | 20 ------------- 5 files changed, 53 deletions(-) delete mode 100644 src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs delete mode 100644 src/gl2gh/Services/IGitlabArchiveDownloader.cs diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 23ebe5fdf..17c03984b 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -2,7 +2,6 @@ using System.CommandLine; using Microsoft.Extensions.DependencyInjection; using OctoshiftCLI.GitlabToGithub.Factories; -using OctoshiftCLI.GitlabToGithub.Services; using OctoshiftCLI.Commands; using OctoshiftCLI.Contracts; using OctoshiftCLI.Extensions; diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index 4a8a2957d..f3e24e32b 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -1,7 +1,6 @@ using System; using System.Runtime.InteropServices; using System.Threading.Tasks; -using OctoshiftCLI.GitlabToGithub.Services; using OctoshiftCLI.Commands; using OctoshiftCLI.Extensions; using OctoshiftCLI.Services; diff --git a/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs b/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs deleted file mode 100644 index 1e498704f..000000000 --- a/src/gl2gh/Factories/GitlabArchiveDownloaderFactory.cs +++ /dev/null @@ -1,30 +0,0 @@ -using OctoshiftCLI.GitlabToGithub.Services; -using OctoshiftCLI.Services; - -namespace OctoshiftCLI.GitlabToGithub.Factories; - -public class GitlabArchiveDownloaderFactory -{ - private readonly OctoLogger _log; - private readonly FileSystemProvider _fileSystemProvider; - private readonly EnvironmentVariableProvider _environmentVariableProvider; - - public GitlabArchiveDownloaderFactory(OctoLogger log, FileSystemProvider fileSystemProvider, EnvironmentVariableProvider environmentVariableProvider) - { - _log = log; - _fileSystemProvider = fileSystemProvider; - _environmentVariableProvider = environmentVariableProvider; - } - - public virtual IGitlabArchiveDownloader CreateSshDownloader(string host, string sshUser, string privateKeyFileFullPath, int sshPort = 22, string bbsSharedHomeDirectory = null) => - new GitlabSshArchiveDownloader(_log, _fileSystemProvider, host, sshUser, privateKeyFileFullPath, sshPort) - { - GitlabSharedHomeDirectory = bbsSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX - }; - - public virtual IGitlabArchiveDownloader CreateSmbDownloader(string host, string smbUser, string smbPassword, string domainName = null, string bbsSharedHomeDirectory = null) => - new GitlabSmbArchiveDownloader(_log, _fileSystemProvider, host, smbUser, smbPassword ?? _environmentVariableProvider.SmbPassword(), domainName) - { - GitlabSharedHomeDirectory = bbsSharedHomeDirectory ?? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS - }; -} diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs index 1c60b4465..7e02505aa 100644 --- a/src/gl2gh/Program.cs +++ b/src/gl2gh/Program.cs @@ -49,7 +49,6 @@ public static async Task Main(string[] args) .AddSingleton() .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) - .AddSingleton() .AddSingleton() .AddHttpClient("Kerberos", kerberos: true, noSsl: false) .AddHttpClient("NoSSL", kerberos: false, noSsl: true) diff --git a/src/gl2gh/Services/IGitlabArchiveDownloader.cs b/src/gl2gh/Services/IGitlabArchiveDownloader.cs deleted file mode 100644 index e929e11c7..000000000 --- a/src/gl2gh/Services/IGitlabArchiveDownloader.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.IO; -using System.Threading.Tasks; -using OctoshiftCLI.Extensions; - -namespace OctoshiftCLI.GitlabToGithub.Services; - -public interface IGitlabArchiveDownloader -{ - const string EXPORT_ARCHIVE_SOURCE_DIRECTORY = "data/migration/export"; - const string DEFAULT_TARGET_DIRECTORY = "bbs_archive_downloads"; - - Task Download(long exportJobId, string targetDirectory = DEFAULT_TARGET_DIRECTORY); - - static string GetSourceExportArchiveAbsolutePath(string bbsSharedHomeDirectory, long exportJobId) => - Path.Join(bbsSharedHomeDirectory, GetSourceExportArchiveRelativePath(exportJobId)).ToUnixPath(); - - static string GetExportArchiveFileName(long exportJobId) => $"Bitbucket_export_{exportJobId}.tar"; - - static string GetSourceExportArchiveRelativePath(long exportJobId) => Path.Join(EXPORT_ARCHIVE_SOURCE_DIRECTORY, GetExportArchiveFileName(exportJobId)).ToUnixPath(); -} From 3e08782a728176786f58b3c214377a421ee1b5b9 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:43:40 -0700 Subject: [PATCH 51/71] Use GITLAB type for GitLab connector. --- src/Octoshift/Services/GithubApi.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 822fb7a43..bf4b2ff53 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -361,7 +361,7 @@ public virtual async Task CreateGitlabMigrationSource(string orgId) name = "GitLab Source", url = "https://not-used", ownerId = orgId, - type = "GITLAB_ARCHIVE" + type = "GITLAB" }, operationName = "createMigrationSource" }; From cb7792ae9ecb9e5f4912353820bba72eb34f7698 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:44:04 -0700 Subject: [PATCH 52/71] Add octoshift_ll_gitlab_self_serve FF in headers. --- src/Octoshift/Services/GithubClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index 6d846bc8d..f31e2a9fc 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -38,7 +38,7 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers if (_httpClient != null) { _httpClient.DefaultRequestHeaders.Add("Accept", "application/vnd.github.v3+json"); - _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api"); + _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api,octoshift_ll_gitlab_self_serve"); _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", personalAccessToken); _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); if (versionProvider?.GetVersionComments() is { } comments) From 86ac89ef7a756696b3a9a8d35a1552eb7589bcab Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:47:41 -0700 Subject: [PATCH 53/71] Add DownloadToFile to GitlabClient. --- src/Octoshift/Services/GitlabClient.cs | 27 +++++++++++++++++++++---- src/gl2gh/Factories/GitlabApiFactory.cs | 16 +++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs index 51af34a56..f7c9176dc 100644 --- a/src/Octoshift/Services/GitlabClient.cs +++ b/src/Octoshift/Services/GitlabClient.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Collections.Generic; using System.Net.Http; using System.Net.Http.Headers; @@ -17,21 +18,23 @@ public class GitlabClient private readonly HttpClient _httpClient; private readonly OctoLogger _log; private readonly RetryPolicy _retryPolicy; + private readonly FileSystemProvider _fileSystemProvider; - public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string gitlabPat) : - this(log, httpClient, versionProvider, retryPolicy) + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string gitlabPat, FileSystemProvider fileSystemProvider) : + this(log, httpClient, versionProvider, retryPolicy, fileSystemProvider) { if (_httpClient != null) { - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("PRIVATE-TOKEN", gitlabPat); + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", gitlabPat); } } - public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy) + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, FileSystemProvider fileSystemProvider) { _log = log; _httpClient = httpClient; _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; if (_httpClient != null) { @@ -126,4 +129,20 @@ private string AddPaginationParams(string url, int start, int limit) return $"{path}?{queryParams}"; } + + public virtual async Task DownloadToFile(string url, string file) + { + _log.LogVerbose($"HTTP GET: {url}"); + + using var response = await _retryPolicy.Retry(async () => + await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead)); + + _log.LogVerbose($"RESPONSE ({response.StatusCode}): "); + + response.EnsureSuccessStatusCode(); + + await using var streamToReadFrom = await response.Content.ReadAsStreamAsync(); + await using var streamToWriteTo = _fileSystemProvider.Open(file, FileMode.Create); + await _fileSystemProvider.CopySourceToTargetStreamAsync(streamToReadFrom, streamToWriteTo); + } } diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs index ae4dad73b..b47574872 100644 --- a/src/gl2gh/Factories/GitlabApiFactory.cs +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -11,14 +11,22 @@ public class GitlabApiFactory private readonly EnvironmentVariableProvider _environmentVariableProvider; private readonly IVersionProvider _versionProvider; private readonly RetryPolicy _retryPolicy; - - public GitlabApiFactory(OctoLogger octoLogger, IHttpClientFactory clientFactory, EnvironmentVariableProvider environmentVariableProvider, IVersionProvider versionProvider, RetryPolicy retryPolicy) + private readonly FileSystemProvider _fileSystemProvider; + + public GitlabApiFactory( + OctoLogger octoLogger, + IHttpClientFactory clientFactory, + EnvironmentVariableProvider environmentVariableProvider, + IVersionProvider versionProvider, + RetryPolicy retryPolicy, + FileSystemProvider fileSystemProvider) { _octoLogger = octoLogger; _clientFactory = clientFactory; _environmentVariableProvider = environmentVariableProvider; _versionProvider = versionProvider; _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; } public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool noSsl = false) @@ -28,7 +36,7 @@ public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool n var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); - var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat, _fileSystemProvider); return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } @@ -37,7 +45,7 @@ public virtual GitlabApi CreateKerberos(string gitlabServerUrl, bool noSsl = fal var httpClient = noSsl ? _clientFactory.CreateClient("KerberosNoSSL") : _clientFactory.CreateClient("Kerberos"); var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); - var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, _fileSystemProvider); return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } } From a8b79ed9f3e4dda1f682958b0e1b3c50464682cd Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:48:09 -0700 Subject: [PATCH 54/71] Add DownloadExportArchive to GitlabApi. --- src/Octoshift/Services/GitlabApi.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index 5726c35e3..9edc404c2 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -54,6 +54,14 @@ public virtual async Task StartExport(string groupPath, string projectPa ); } + public virtual async Task DownloadExportArchive(string groupPath, string projectPath, string file) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export/download"; + + await _client.DownloadToFile(url, file); + } + public virtual async Task> GetProjects(string groupPath) { var encodedGroupPath = Uri.EscapeDataString(groupPath); From 5a3f518cac5bbcf2bad4da31a3cc25e47917ad97 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:54:28 -0700 Subject: [PATCH 55/71] Download GitLab archives with HttpDownloadService. --- .../MigrateRepo/MigrateRepoCommand.cs | 9 ++- .../MigrateRepo/MigrateRepoCommandHandler.cs | 69 +++++-------------- 2 files changed, 23 insertions(+), 55 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 17c03984b..8a2abc5c3 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -162,9 +162,11 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar var log = sp.GetRequiredService(); var environmentVariableProvider = sp.GetRequiredService(); var fileSystemProvider = sp.GetRequiredService(); + var httpDownloadServiceFactory = sp.GetRequiredService(); + var httpDownloadService = args.NoSslVerify ? httpDownloadServiceFactory.CreateClientNoSsl() : httpDownloadServiceFactory.CreateDefault(); + GithubApi githubApi = null; GitlabApi gitlabApi = null; - IGitlabArchiveDownloader gitlabArchiveDownloader = null; AzureApi azureApi = null; AwsApi awsApi = null; @@ -183,9 +185,6 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); } - var gitlabArchiveDownloaderFactory = sp.GetRequiredService(); - gitlabArchiveDownloader = gitlabArchiveDownloaderFactory.CreateDownloader(); - var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); if (azureStorageConnectionString.HasValue()) { @@ -201,6 +200,6 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar var warningsCountLogger = sp.GetRequiredService(); - return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, gitlabArchiveDownloader, azureApi, awsApi, fileSystemProvider, warningsCountLogger); + return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, azureApi, awsApi, httpDownloadService, fileSystemProvider, warningsCountLogger); } } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index f3e24e32b..7f486871b 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -15,7 +15,7 @@ public class MigrateRepoCommandHandler : ICommandHandler private readonly AzureApi _azureApi; private readonly AwsApi _awsApi; private readonly EnvironmentVariableProvider _environmentVariableProvider; - private readonly IGitlabArchiveDownloader _bbsArchiveDownloader; + private readonly HttpDownloadService _httpDownloadService; private readonly FileSystemProvider _fileSystemProvider; private readonly WarningsCountLogger _warningsCountLogger; private const int CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS = 10000; @@ -26,9 +26,9 @@ public MigrateRepoCommandHandler( GithubApi githubApi, GitlabApi gitlabApi, EnvironmentVariableProvider environmentVariableProvider, - IGitlabArchiveDownloader bbsArchiveDownloader, AzureApi azureApi, AwsApi awsApi, + HttpDownloadService httpDownloadService, FileSystemProvider fileSystemProvider, WarningsCountLogger warningsCountLogger) { @@ -37,8 +37,8 @@ public MigrateRepoCommandHandler( _gitlabApi = gitlabApi; _azureApi = azureApi; _awsApi = awsApi; + _httpDownloadService = httpDownloadService; _environmentVariableProvider = environmentVariableProvider; - _bbsArchiveDownloader = bbsArchiveDownloader; _fileSystemProvider = fileSystemProvider; _warningsCountLogger = warningsCountLogger; } @@ -52,7 +52,6 @@ public async Task Handle(MigrateRepoCommandArgs args) ValidateOptions(args); - var exportId = 0L; var migrationSourceId = ""; if (args.ShouldImportArchive()) @@ -69,27 +68,18 @@ public async Task Handle(MigrateRepoCommandArgs args) if (args.ShouldGenerateArchive()) { - exportId = await GenerateArchive(args); + await GenerateArchive(args); - if (args.ShouldDownloadArchive()) - { - args.ArchivePath = await DownloadArchive(exportId); - } + _log.LogInformation($"Downloading GitLab archive..."); - if (!args.ShouldDownloadArchive() && args.ShouldUploadArchive()) - { - _log.LogWarning($"You haven't specified --ssh-user or --smb-user, so we assume that you're running the CLI on the Bitbucket instance itself. If you are not running this command on the Bitbucket instance, run this command again with the --ssh-user or --smb-user argument to allow the CLI to download the migration archive from the server."); - } + args.ArchivePath ??= _fileSystemProvider.GetTempFileName(); + await _gitlabApi.DownloadExportArchive(args.GitlabGroup, args.GitlabProject, args.ArchivePath); + + _log.LogInformation(args.KeepArchive ? $"Archive downloaded to \"{args.ArchivePath}\"" : "Archive download complete"); } if (args.ShouldUploadArchive()) { - // This is for the case where the CLI is being run on the BBS server itself - if (args.ArchivePath.IsNullOrWhiteSpace()) - { - args.ArchivePath = GetSourceExportArchiveAbsolutePath(args.GitlabSharedHome, exportId); - } - _log.LogInformation($"Archive path: {args.ArchivePath}"); try @@ -112,7 +102,7 @@ public async Task Handle(MigrateRepoCommandArgs args) } finally { - if (!args.KeepArchive && args.ShouldDownloadArchive()) + if (!args.KeepArchive) { DeleteArchive(args.ArchivePath); } @@ -125,18 +115,6 @@ public async Task Handle(MigrateRepoCommandArgs args) } } - private string GetSourceExportArchiveAbsolutePath(string bbsSharedHomeDirectory, long exportId) - { - if (bbsSharedHomeDirectory.IsNullOrWhiteSpace()) - { - bbsSharedHomeDirectory = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS - : GitlabSettings.DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX; - } - - return IGitlabArchiveDownloader.GetSourceExportArchiveAbsolutePath(bbsSharedHomeDirectory, exportId); - } - private void DeleteArchive(string path) { try @@ -152,38 +130,29 @@ private void DeleteArchive(string path) } } - private async Task DownloadArchive(long exportId) - { - _log.LogInformation($"Download archive {exportId} started..."); - var downloadedArchiveFullPath = await _bbsArchiveDownloader.Download(exportId); - _log.LogInformation($"Archive was successfully downloaded at \"{downloadedArchiveFullPath}\"."); - - return downloadedArchiveFullPath; - } - - private async Task GenerateArchive(MigrateRepoCommandArgs args) + private async Task GenerateArchive(MigrateRepoCommandArgs args) { - var exportId = await _gitlabApi.StartExport(args.GitlabGroup, args.GitlabProject); + await _gitlabApi.StartExport(args.GitlabGroup, args.GitlabProject); - _log.LogInformation($"Export started. Export ID: {exportId}"); + _log.LogInformation($"Export started."); - var (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + var (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); while (ExportState.IsInProgress(exportState)) { - _log.LogInformation($"Export status: {exportState}; {exportProgress}% complete"); + _log.LogInformation($"Export status: {exportState}."); await Task.Delay(CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS); - (exportState, exportMessage, exportProgress) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); } if (ExportState.IsError(exportState)) { - throw new OctoshiftCliException($"Bitbucket export failed --> State: {exportState}; Message: {exportMessage}"); + throw new OctoshiftCliException($"GitLab archive export failed!"); } - _log.LogInformation($"Export completed. Your migration archive should be ready **on your Bitbucket instance** at $BITBUCKET_SHARED_HOME/data/migration/export/Bitbucket_export_{exportId}.tar"); + _log.LogInformation($"Archive export completed."); - return exportId; + return archiveUrl; } private async Task UploadArchiveToAzure(string archivePath) From b5a233744e5ad99ddc84bde4094cc2d991c79b54 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:55:32 -0700 Subject: [PATCH 56/71] Remove Samba and SSH args from MigrateRepoCommandArgs. --- .../MigrateRepo/MigrateRepoCommandArgs.cs | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index ffc768c94..6ff8403dc 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -37,19 +37,8 @@ public class MigrateRepoCommandArgs : CommandArgs public string GitlabProject { get; set; } [Secret] public string GitlabPat { get; set; } - public string GitlabSharedHome { get; set; } public bool NoSslVerify { get; set; } - public string ArchiveDownloadHost { get; set; } - public string SshUser { get; set; } - public string SshPrivateKey { get; set; } - public int SshPort { get; set; } = 22; - - public string SmbUser { get; set; } - [Secret] - public string SmbPassword { get; set; } - public string SmbDomain { get; set; } - public bool KeepArchive { get; set; } public bool UseGithubStorage { get; set; } @@ -68,7 +57,6 @@ public override void Validate(OctoLogger log) if (ShouldGenerateArchive()) { ValidateGenerateOptions(); - ValidateDownloadOptions(); } else { @@ -84,11 +72,6 @@ public override void Validate(OctoLogger log) { ValidateImportOptions(); } - - if (SshPort == 7999) - { - log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); - } } private void ValidateNoGenerateOptions() @@ -102,17 +85,10 @@ private void ValidateNoGenerateOptions() { throw new OctoshiftCliException("--no-ssl-verify cannot be provided with --archive-path or --archive-url."); } - - if (new[] { SshUser, SshPrivateKey, ArchiveDownloadHost, SmbUser, SmbPassword, SmbDomain }.Any(obj => obj.HasValue())) - { - throw new OctoshiftCliException("SSH or SMB download options cannot be provided with --archive-path or --archive-url."); - } } public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchivePath.HasValue() && !ArchiveUrl.HasValue(); - public bool ShouldDownloadArchive() => SshUser.HasValue() || SmbUser.HasValue(); - public bool ShouldUploadArchive() => ArchiveUrl.IsNullOrWhiteSpace() && GithubOrg.HasValue(); // NOTE: ArchiveUrl doesn't necessarily refer to the value passed in by the user to the CLI - it is set during CLI runtime when an archive is uploaded to blob storage @@ -131,29 +107,6 @@ private void ValidateGenerateOptions() } } - private void ValidateDownloadOptions() - { - var sshArgs = new[] { SshUser, SshPrivateKey }; - var smbArgs = new[] { SmbUser, SmbPassword }; - var shouldUseSsh = sshArgs.Any(arg => arg.HasValue()); - var shouldUseSmb = smbArgs.Any(arg => arg.HasValue()); - - if (shouldUseSsh && shouldUseSmb) - { - throw new OctoshiftCliException("You can't provide both SSH and SMB credentials together."); - } - - if (SshUser.HasValue() ^ SshPrivateKey.HasValue()) - { - throw new OctoshiftCliException("Both --ssh-user and --ssh-private-key must be specified for SSH download."); - } - - if (ArchiveDownloadHost.HasValue() && !shouldUseSsh && !shouldUseSmb) - { - throw new OctoshiftCliException("--archive-download-host can only be provided if SSH or SMB download options are provided."); - } - } - private void ValidateUploadOptions() { if (AwsBucketName.IsNullOrWhiteSpace() && new[] { AwsAccessKey, AwsSecretKey, AwsSessionToken, AwsRegion }.Any(x => x.HasValue())) From 4492c198be9a8e365d6312835ac31db6f45107a8 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 19:56:08 -0700 Subject: [PATCH 57/71] Remove Samba and SSH flow from GL GenerateScriptCommand. --- .../GenerateScript/GenerateScriptCommand.cs | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs index 4b07976d3..6393aa515 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -20,13 +20,7 @@ public GenerateScriptCommand() : base( AddOption(TargetApiUrl); AddOption(GitlabPat); AddOption(GitlabProject); - AddOption(GitlabSharedHome); - AddOption(SshUser); - AddOption(SshPrivateKey); - AddOption(SshPort); AddOption(ArchiveDownloadHost); - AddOption(SmbUser); - AddOption(SmbDomain); AddOption(Output); AddOption(Kerberos); AddOption(Verbose); @@ -52,38 +46,10 @@ public GenerateScriptCommand() : base( name: "--bbs-project", description: "The Bitbucket project to migrate. If not set will migrate all projects."); - public Option GitlabSharedHome { get; } = new( - name: "--bbs-shared-home", - description: "Bitbucket server's shared home directory. Defaults to \"/var/atlassian/application-data/bitbucket/shared\" if downloading the archive from a server using SSH " + - "and \"c$\\atlassian\\applicationdata\\bitbucket\\shared\" if downloading using SMB."); - public Option ArchiveDownloadHost { get; } = new( name: "--archive-download-host", description: "The host to use to connect to the Bitbucket Server/Data Center instance via SSH or SMB. Defaults to the host from the Bitbucket Server URL (--bbs-server-url)."); - public Option SshUser { get; } = new( - name: "--ssh-user", - description: "The SSH user to be used for downloading the export archive off of the Bitbucket server."); - - public Option SshPrivateKey { get; } = new( - name: "--ssh-private-key", - description: "The full path of the private key file to be used for downloading the export archive off of the Bitbucket Server using SSH/SFTP."); - - public Option SshPort { get; } = new( - name: "--ssh-port", - description: "The SSH port (default: 22).", - getDefaultValue: () => 22); - - public Option SmbUser { get; } = new( - name: "--smb-user", - description: "The SMB user used for authentication when downloading the export archive from the Bitbucket Server instance." + - $"{Environment.NewLine}" + - "Note: You must also specify the SMB password using the SMB_PASSWORD environment variable."); - - public Option SmbDomain { get; } = new( - name: "--smb-domain", - description: "The optional domain name when using SMB for downloading the export archive."); - public Option GithubOrg { get; } = new("--github-org") { IsRequired = true }; From 2cc70c6b5441b7d6f71f5c8893c83dbb6dab3e66 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 20:03:40 -0700 Subject: [PATCH 58/71] Remove GitlabUsername from GL InventoryReportCommandArgs. --- src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs index eb6f40e95..e7ba94096 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -6,7 +6,6 @@ public class InventoryReportCommandArgs : CommandArgs { public string GitlabServerUrl { get; set; } public string GitlabGroup { get; set; } - public string GitlabUsername { get; set; } [Secret] public string GitlabPat { get; set; } public bool NoSslVerify { get; set; } From c245bd28b8f145038af0dd4406e9230b5c439081 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 20:10:47 -0700 Subject: [PATCH 59/71] Always include archived status in projects CSVs. --- .../Commands/InventoryReport/InventoryReportCommand.cs | 2 +- src/gl2gh/Services/ProjectsCsvGeneratorService.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index d715ff3e2..666669559 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -43,7 +43,7 @@ public InventoryReportCommand() : base( public Option Minimal { get; } = new( name: "--minimal", - description: "Significantly speeds up the generation of the CSV files by including the bare minimum info. Will omit the archived state and PR count for repos and the PR count for projects."); + description: "Omit the MR count from group and project reports for quicker report generation."); public Option Verbose { get; } = new("--verbose"); diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs index 59ba61fe2..4e4f3a93b 100644 --- a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -24,8 +24,8 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); var result = new StringBuilder(); - result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"); - result.AppendLine(!minimal ? ",is-archived,mr-count" : null); + result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"); + result.AppendLine(!minimal ? ",mr-count" : null); var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; @@ -43,14 +43,14 @@ public virtual async Task Generate(string gitlabServerUrl, string gitlab if (lastCommitDate == null) { - result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\""); + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); } else { - result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\""); + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); } - result.AppendLine(!minimal ? $",\"{project.Archived}\",{mrCount}" : null); + result.AppendLine(!minimal ? $",{mrCount}" : null); } } From 7fe146f432d8bc7e322f9ecd0302a446795ae49b Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 20:12:06 -0700 Subject: [PATCH 60/71] Use GitLab terminology in GL InventoryReportCommand. --- .../Commands/InventoryReport/InventoryReportCommand.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs index 666669559..fc165742a 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -11,7 +11,7 @@ public class InventoryReportCommand : CommandBase GitlabGroup { get; } = new( name: "--gitlab-group", - description: "The Bitbucket project key. If not provided will iterate over all projects that the user has access to."); + description: "The GitLab group. Iterates over all projects that the user has access to if not provided."); public Option GitlabPat { get; } = new( name: "--gitlab-pat", - description: "The Bitbucket password of the user specified by --gitlab-username. If not set will be read from BBS_PASSWORD environment variable."); + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); public Option NoSslVerify { get; } = new( name: "--no-ssl-verify", - description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. " + - "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow data to be extracted."); + description: "Disables SSL verification when communicating with your GitLab instance. " + + "If your GitLab instance has a self-signed SSL certificate then setting this flag will allow data to be extracted."); public Option Minimal { get; } = new( name: "--minimal", From 80441cb246f3f78190186cd78be0f23b42092cd8 Mon Sep 17 00:00:00 2001 From: Maxwell Pray Date: Wed, 13 May 2026 20:13:06 -0700 Subject: [PATCH 61/71] Remove Samba and shared home logic from GL MigrateRepoCommandHandler. --- .../MigrateRepo/MigrateRepoCommandHandler.cs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index 7f486871b..a9de56b87 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -277,8 +277,6 @@ private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => a private string GetGitlabPat(MigrateRepoCommandArgs args) => args.GitlabPat.HasValue() ? args.GitlabPat : _environmentVariableProvider.GitlabPat(false); - private string GetSmbPassword(MigrateRepoCommandArgs args) => args.SmbPassword.HasValue() ? args.SmbPassword : _environmentVariableProvider.SmbPassword(false); - private string GetGitlabProjectUrl(MigrateRepoCommandArgs args) { return args.GitlabServerUrl.HasValue() && args.GitlabGroup.HasValue() && args.GitlabProject.HasValue() @@ -297,17 +295,6 @@ private void ValidateOptions(MigrateRepoCommandArgs args) throw new OctoshiftCliException("BBS password must be either set as BBS_PAT environment variable or passed as --bbs-pat."); } } - - if ((args.SmbUser.HasValue() && GetSmbPassword(args).IsNullOrWhiteSpace()) || (args.SmbPassword.HasValue() && args.SmbUser.IsNullOrWhiteSpace())) - { - throw new OctoshiftCliException("Both --smb-user and --smb-password (or SMB_PASSWORD env. variable) must be specified for SMB download."); - } - - // Validate --bbs-shared-home if running on Bitbucket instance (not using SSH/SMB) - if (!args.ShouldDownloadArchive() && args.GitlabSharedHome.HasValue() && !_fileSystemProvider.DirectoryExists(args.GitlabSharedHome)) - { - throw new OctoshiftCliException($"The path provided for --bbs-shared-home does not exist or is not accessible: {args.GitlabSharedHome}"); - } } // Validate --archive-path if provided From dc467e4ebcc674738f44be5612d63376bdac4238 Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 16:42:08 +0000 Subject: [PATCH 62/71] delete stale tests and unused gitlabsettings --- .../GitlabSmbArchiveDownloaderTests.cs | 191 ------------------ .../GitlabSshArchiveDownloaderTests.cs | 85 -------- .../Services/ReposCsvGeneratorServiceTests.cs | 186 ----------------- src/gl2gh/GitlabSettings.cs | 8 - 4 files changed, 470 deletions(-) delete mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs delete mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs delete mode 100644 src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs delete mode 100644 src/gl2gh/GitlabSettings.cs diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs deleted file mode 100644 index 135fd774f..000000000 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSmbArchiveDownloaderTests.cs +++ /dev/null @@ -1,191 +0,0 @@ -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Moq; -using OctoshiftCLI.GitlabToGithub.Services; -using OctoshiftCLI.Extensions; -using OctoshiftCLI.Services; -using SMBLibrary; -using SMBLibrary.Client; -using Xunit; -using FileAttributes = SMBLibrary.FileAttributes; - -namespace OctoshiftCLI.Tests.bbs2gh.Services; - -public class GitlabSmbArchiveDownloaderTests -{ - private const int EXPORT_JOB_ID = 1; - private const string SHARE_ROOT = "SHARE_ROOT"; - private const string BBS_HOME_DIRECTORY_FROM_SHARE = "PATH\\TO\\BBS\\HOME\\DIRECTORY"; - private const string BBS_HOME_DIRECTORY = $"{SHARE_ROOT}\\{BBS_HOME_DIRECTORY_FROM_SHARE}"; - private const string TARGET_DIRECTORY = "TARGET"; - private const string HOST = "HOST"; - private const string SMB_USER = "SMB_USER"; - private const string SMB_PASSWORD = "SMB_PASSWORD"; - private const string DOMAIN = "DOMAIN"; - - private readonly string _exportArchiveFilename = $"Bitbucket_export_{EXPORT_JOB_ID}.tar"; - private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); - private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); - private readonly Mock _mockSmbClient = new(); - private readonly Mock _mockSmbFileStore = new(); - private readonly GitlabSmbArchiveDownloader _bbsArchiveDownloader; - - public GitlabSmbArchiveDownloaderTests() - { - _bbsArchiveDownloader = new GitlabSmbArchiveDownloader( - _mockOctoLogger.Object, - _mockFileSystemProvider.Object, - _mockSmbClient.Object, - HOST, - SMB_USER, - SMB_PASSWORD, - DOMAIN) - { GitlabSharedHomeDirectory = BBS_HOME_DIRECTORY }; - } - - [Fact] - public async Task Download_Returns_Downloaded_Archive_Full_Name() - { - // Arrange - var expectedSourceArchiveFullNameAfterShare = Path.Join(BBS_HOME_DIRECTORY_FROM_SHARE, "data/migration/export", _exportArchiveFilename).ToWindowsPath(); - var expectedTargetArchiveFullName = Path.Join(TARGET_DIRECTORY, _exportArchiveFilename).ToUnixPath(); - - _mockSmbClient.Setup(m => m.Connect(HOST, SMBTransportType.DirectTCPTransport)).Returns(true); - _mockSmbClient.Setup(m => m.Login(DOMAIN, SMB_USER, SMB_PASSWORD)).Returns(NTStatus.STATUS_SUCCESS); - var createSmbFileStoreStatus = NTStatus.STATUS_SUCCESS; - _mockSmbClient.Setup(m => m.TreeConnect(SHARE_ROOT, out createSmbFileStoreStatus)).Returns(_mockSmbFileStore.Object); - - var sharedFileHandle = new object(); - var fileStatus = FileStatus.FILE_OPENED; - _mockSmbFileStore.Setup(m => m.CreateFile( - out sharedFileHandle, - out fileStatus, - expectedSourceArchiveFullNameAfterShare, - AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.Read, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null)) - .Returns(NTStatus.STATUS_SUCCESS); - - FileInformation fileStandardInformation = new FileStandardInformation - { - AllocationSize = 10 * 1024 * 1024 // 10 MB - }; - _mockSmbFileStore - .Setup(m => m.GetFileInformation(out fileStandardInformation, sharedFileHandle, FileInformationClass.FileStandardInformation)) - .Returns(NTStatus.STATUS_SUCCESS); - - var data = new byte[1024]; - _mockSmbFileStore - .SetupSequence(m => m.ReadFile(out data, sharedFileHandle, It.IsAny(), It.IsAny())) - .Returns(NTStatus.STATUS_SUCCESS) - .Returns(NTStatus.STATUS_SUCCESS) - .Returns(NTStatus.STATUS_END_OF_FILE); - - // Act - var actualTargetArchiveFullName = await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); - - // Assert - _mockSmbClient.Verify(m => m.Connect(HOST, SMBTransportType.DirectTCPTransport), Times.Once); - _mockSmbClient.Verify(m => m.Login(DOMAIN, SMB_USER, SMB_PASSWORD), Times.Once); - _mockSmbClient.Verify(m => m.TreeConnect(SHARE_ROOT, out createSmbFileStoreStatus), Times.Once); - _mockSmbFileStore.Verify(m => m.CreateFile( - out sharedFileHandle, - out fileStatus, - expectedSourceArchiveFullNameAfterShare, - AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.Read, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null), - Times.Once); - _mockSmbFileStore.Verify(m => m.GetFileInformation(out fileStandardInformation, sharedFileHandle, FileInformationClass.FileStandardInformation), Times.Once); - _mockSmbFileStore.Verify(m => m.ReadFile(out data, sharedFileHandle, It.IsAny(), It.IsAny()), Times.Exactly(3)); - _mockFileSystemProvider.Verify(m => m.CreateDirectory(TARGET_DIRECTORY), Times.Once); - _mockFileSystemProvider.Verify(m => m.Open(expectedTargetArchiveFullName, FileMode.Create), Times.Once); - _mockFileSystemProvider.Verify(m => m.WriteAsync(It.IsAny(), data, It.IsAny()), Times.Exactly(2)); - - actualTargetArchiveFullName.Should().Be(expectedTargetArchiveFullName); - } - - [Fact] - public async Task Download_Throws_When_Cannot_Connect_To_Host() - { - // Arrange - _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(false); - - // Act, Assert - await _bbsArchiveDownloader - .Invoking(async x => await x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) - .Should() - .ThrowExactlyAsync(); - } - - [Fact] - public async Task Download_Throws_When_Cannot_Login() - { - // Arrange - _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); - _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_LOGON_FAILURE); - - // Act, Assert - await _bbsArchiveDownloader - .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) - .Should() - .ThrowExactlyAsync() - .WithMessage($"*{NTStatus.STATUS_LOGON_FAILURE}*"); - } - - [Fact] - public async Task Download_Throws_When_Cannot_Connect_To_Share() - { - // Arrange - _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); - _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_SUCCESS); - var status = NTStatus.STATUS_BAD_NETWORK_NAME; - _mockSmbClient.Setup(m => m.TreeConnect(It.IsAny(), out status)).Returns(_mockSmbFileStore.Object); - - // Act, Assert - await _bbsArchiveDownloader - .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) - .Should() - .ThrowExactlyAsync() - .WithMessage($"*{NTStatus.STATUS_BAD_NETWORK_NAME}*"); - } - - [Fact] - public async Task Download_Throws_When_Source_Export_Archive_Does_Not_Exist() - { - // Arrange - _mockSmbClient.Setup(m => m.Connect(It.IsAny(), SMBTransportType.DirectTCPTransport)).Returns(true); - _mockSmbClient.Setup(m => m.Login(It.IsAny(), It.IsAny(), It.IsAny())).Returns(NTStatus.STATUS_SUCCESS); - var status = NTStatus.STATUS_SUCCESS; - _mockSmbClient.Setup(m => m.TreeConnect(It.IsAny(), out status)).Returns(_mockSmbFileStore.Object); - - object sharedFileHandle; - FileStatus fileStatus; - _mockSmbFileStore.Setup(m => m.CreateFile( - out sharedFileHandle, - out fileStatus, - It.IsAny(), - AccessMask.GENERIC_READ | AccessMask.SYNCHRONIZE, - FileAttributes.Normal, - ShareAccess.Read, - CreateDisposition.FILE_OPEN, - CreateOptions.FILE_NON_DIRECTORY_FILE | CreateOptions.FILE_SYNCHRONOUS_IO_ALERT, - null)) - .Returns(NTStatus.STATUS_OBJECT_NAME_NOT_FOUND); - - // Act, Assert - await _bbsArchiveDownloader - .Invoking(x => x.Download(EXPORT_JOB_ID, TARGET_DIRECTORY)) - .Should() - .ThrowExactlyAsync() - .WithMessage($"*{NTStatus.STATUS_OBJECT_NAME_NOT_FOUND}*"); - } -} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs deleted file mode 100644 index 94e407df5..000000000 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabSshArchiveDownloaderTests.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; -using FluentAssertions; -using Moq; -using OctoshiftCLI.GitlabToGithub.Services; -using OctoshiftCLI.Services; -using Renci.SshNet; -using Xunit; - -namespace OctoshiftCLI.Tests.bbs2gh.Services; - -public sealed class GitlabSshArchiveDownloaderTests : IDisposable -{ - private const int EXPORT_JOB_ID = 1; - private const string BBS_HOME_DIRECTORY = "BBS_HOME"; - private const string TARGET_DIRECTORY = "TARGET"; - - private readonly string _exportArchiveFilename = $"Bitbucket_export_{EXPORT_JOB_ID}.tar"; - private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); - private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); - private readonly Mock _mockSftpClient = new(); - private readonly GitlabSshArchiveDownloader _bbsArchiveDownloader; - - public GitlabSshArchiveDownloaderTests() - { - _bbsArchiveDownloader = new GitlabSshArchiveDownloader(_mockOctoLogger.Object, _mockFileSystemProvider.Object, _mockSftpClient.Object) - { - GitlabSharedHomeDirectory = BBS_HOME_DIRECTORY - }; - - _mockSftpClient.Setup(m => m.Exists(It.IsAny())).Returns(true); - - var mockAsyncResult = new Mock(); - mockAsyncResult.Setup(m => m.IsCompleted).Returns(true); - _mockSftpClient - .Setup(m => m.BeginDownloadFile(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) - .Returns(mockAsyncResult.Object); - } - - [Fact] - public async Task Download_Returns_Downloaded_Archive_Full_Name() - { - // Arrange - var expectedSourceArchiveFullName = Path.Join(BBS_HOME_DIRECTORY, "data/migration/export", _exportArchiveFilename).Replace('\\', '/'); - var expectedTargetArchiveFullName = Path.Join(TARGET_DIRECTORY, _exportArchiveFilename).Replace('\\', '/'); - - // Act - var actualDownloadedArchiveFullName = await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); - - // Assert - _mockSftpClient.Verify(m => - m.BeginDownloadFile( - expectedSourceArchiveFullName, - It.IsAny(), - null, - null, - It.IsAny>())); - - _mockFileSystemProvider.Verify(m => m.Open(expectedTargetArchiveFullName, FileMode.Create)); - actualDownloadedArchiveFullName.Should().Be(expectedTargetArchiveFullName); - } - - [Fact] - public async Task Download_Throws_When_Source_Export_Archive_Does_Not_Exist() - { - // Arrange - _mockSftpClient.Setup(m => m.Exists(It.IsAny())).Returns(false); - - // Act, Assert - await _bbsArchiveDownloader.Invoking(x => x.Download(EXPORT_JOB_ID)).Should().ThrowExactlyAsync(); - } - - [Fact] - public async Task Download_Creates_Target_Directory() - { - // Arrange, Act - await _bbsArchiveDownloader.Download(EXPORT_JOB_ID, TARGET_DIRECTORY); - - // Assert - _mockFileSystemProvider.Verify(m => m.CreateDirectory(TARGET_DIRECTORY), Times.Once); - } - - public void Dispose() => _bbsArchiveDownloader?.Dispose(); -} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs deleted file mode 100644 index 87af627f6..000000000 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/ReposCsvGeneratorServiceTests.cs +++ /dev/null @@ -1,186 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using FluentAssertions; -using Moq; -using Octoshift.Models; -using OctoshiftCLI.GitlabToGithub; -using OctoshiftCLI.GitlabToGithub.Factories; -using OctoshiftCLI.Services; -using Xunit; - -namespace OctoshiftCLI.Tests.GitlabToGithub.Commands -{ - public class ReposCsvGeneratorServiceTests - { - private const string FULL_CSV_HEADER = "project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived,pr-count"; - private const string MINIMAL_CSV_HEADER = "project-key,project-name,repo,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes"; - - private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); - - private const string BBS_SERVER_URL = "http://bbs-server-url"; - private const string BBS_FOO_PROJECT = "project"; - private const string BBS_FOO_PROJECT_KEY = "FP"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; - private const bool NO_SSL_VERIFY = true; - private readonly (string, string) _bbsProject = (BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT); - private const string BBS_REPO = "foo-repo"; - private const string BBS_REPO_SLUG = "foo-repo-slug"; - private const bool ARCHIVED = false; - private const ulong REPO_SIZE = 10000UL; - private const ulong ATTACHMENTS_SIZE = 10000UL; - private readonly IEnumerable _bbsRepos = [new() { Name = BBS_REPO, Slug = BBS_REPO_SLUG }]; - - private readonly ReposCsvGeneratorService _service; - - public ReposCsvGeneratorServiceTests() - { - _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); - _service = new ReposCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); - } - - [Fact] - public async Task Generate_Should_Return_Correct_Csv_For_One_Repo() - { - // Arrange - var prCount = 822; - var lastCommitDate = DateTime.Now; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); - _mockGitlabInspectorService.Setup(m => m.GetRepositoryPullRequestCount(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(prCount); - _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); - _mockGitlabApi.Setup(m => m.GetIsRepositoryArchived(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(ARCHIVED); - _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); - - // Assert - var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\",\"False\",{prCount}{Environment.NewLine}"; - - result.Should().Be(expected); - } - - [Fact] - public async Task Generate_Should_Return_Correct_Csv_For_One_Repo_Without_Archived_Field_For_Outdated_BBS_Version() - { - // Arrange - var prCount = 822; - var lastCommitDate = DateTime.Now; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); - _mockGitlabInspectorService.Setup(m => m.GetRepositoryPullRequestCount(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(prCount); - _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); - _mockGitlabApi.Setup(m => m.GetIsRepositoryArchived(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(ARCHIVED); - _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); - - // Assert - var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\",\"False\",{prCount}{Environment.NewLine}"; - - result.Should().Be(expected); - } - - [Fact] - public async Task Generate_Should_Return_Minimal_Csv_When_Minimal_Is_True() - { - // Arrange - var lastCommitDate = DateTime.Now; - const bool minimal = true; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsRepos); - _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); - _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); - - // Assert - var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_REPO}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; - - result.Should().Be(expected); - _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); - } - - [Fact] - public async Task Generate_Should_Include_Empty_Entry_For_Null_Latest_Commit_Date() - { - // Arrange - const bool minimal = true; - - var project_name = "project,name"; - var repo_name = "repo,name"; - var expected_project_name = "project%2Cname"; - var expected_repo_name = "repo%2Cname"; - var bbsProject = (BBS_FOO_PROJECT_KEY, project_name); - var bbsRepos = new List { new() { Name = repo_name, Slug = BBS_REPO_SLUG } }; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsRepos); - _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)); - _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); - - // Assert - var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{expected_project_name}\",\"{expected_repo_name}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",,\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; - - result.Should().Be(expected); - _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); - } - - [Fact] - public async Task Generate_Should_Escape_Project_And_Repo_Names() - { - // Arrange - var lastCommitDate = DateTime.Now; - const bool minimal = true; - - var project_name = "project,name"; - var repo_name = "repo,name"; - var expected_project_name = "project%2Cname"; - var expected_repo_name = "repo%2Cname"; - var bbsProject = (BBS_FOO_PROJECT_KEY, project_name); - var bbsRepos = new List { new() { Name = repo_name, Slug = BBS_REPO_SLUG } }; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(bbsRepos); - _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG)).ReturnsAsync(lastCommitDate); - _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(BBS_FOO_PROJECT_KEY, BBS_REPO_SLUG, BBS_USERNAME, BBS_PASSWORD)).ReturnsAsync((REPO_SIZE, ATTACHMENTS_SIZE)); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY, minimal); - - // Assert - var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{expected_project_name}\",\"{expected_repo_name}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}/repos/{BBS_REPO_SLUG}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{REPO_SIZE:D}\",\"{ATTACHMENTS_SIZE:D}\"{Environment.NewLine}"; - - result.Should().Be(expected); - _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); - } - } -} diff --git a/src/gl2gh/GitlabSettings.cs b/src/gl2gh/GitlabSettings.cs deleted file mode 100644 index 0b76fd1a0..000000000 --- a/src/gl2gh/GitlabSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace OctoshiftCLI.GitlabToGithub; - -public static class GitlabSettings -{ - public const string DEFAULT_BBS_SHARED_HOME_DIRECTORY_WINDOWS = "c$\\atlassian\\applicationdata\\bitbucket\\shared"; - - public const string DEFAULT_BBS_SHARED_HOME_DIRECTORY_LINUX = "/var/atlassian/application-data/bitbucket/shared"; -} From edf51a9e1a76acc4be28efbad257a15990484f8d Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 16:45:29 +0000 Subject: [PATCH 63/71] remove unused packagereferences in csproj --- src/gl2gh/gl2gh.csproj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/gl2gh/gl2gh.csproj b/src/gl2gh/gl2gh.csproj index af534add2..2088f3769 100644 --- a/src/gl2gh/gl2gh.csproj +++ b/src/gl2gh/gl2gh.csproj @@ -12,8 +12,6 @@ - - From ebd01dae7578090aa1ed6f1667d7c290901821b2 Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 17:29:43 +0000 Subject: [PATCH 64/71] remove Kerberos --- .../Commands/GenerateScript/GenerateScriptCommand.cs | 4 +--- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs | 4 +--- src/gl2gh/Factories/GitlabApiFactory.cs | 9 --------- src/gl2gh/Program.cs | 9 +++------ 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs index 6393aa515..2b1b8d61d 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -111,9 +111,7 @@ public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandA var environmentVariableProvider = sp.GetRequiredService(); var gitlabApiFactory = sp.GetRequiredService(); - var gitlabApi = args.Kerberos - ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi, environmentVariableProvider); } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index 8a2abc5c3..caa0ac2c7 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -180,9 +180,7 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar { var gitlabApiFactory = sp.GetRequiredService(); - gitlabApi = args.Kerberos - ? gitlabApiFactory.CreateKerberos(args.GitlabServerUrl, args.NoSslVerify) - : gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); } var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs index b47574872..b5c4ad3af 100644 --- a/src/gl2gh/Factories/GitlabApiFactory.cs +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -39,13 +39,4 @@ public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool n var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat, _fileSystemProvider); return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); } - - public virtual GitlabApi CreateKerberos(string gitlabServerUrl, bool noSsl = false) - { - var httpClient = noSsl ? _clientFactory.CreateClient("KerberosNoSSL") : _clientFactory.CreateClient("Kerberos"); - - var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); - var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, _fileSystemProvider); - return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); - } } diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs index 7e02505aa..dd9e85cb8 100644 --- a/src/gl2gh/Program.cs +++ b/src/gl2gh/Program.cs @@ -50,9 +50,7 @@ public static async Task Main(string[] args) .AddSingleton() .AddSingleton(sp => sp.GetRequiredService()) .AddSingleton() - .AddHttpClient("Kerberos", kerberos: true, noSsl: false) - .AddHttpClient("NoSSL", kerberos: false, noSsl: true) - .AddHttpClient("KerberosNoSSL", kerberos: true, noSsl: true) + .AddHttpClient("NoSSL", noSsl: true) .AddHttpClient("Default"); var serviceProvider = serviceCollection.BuildServiceProvider(); @@ -141,11 +139,10 @@ private static async Task LatestVersionCheck(ServiceProvider sp) } } - private static IServiceCollection AddHttpClient(this IServiceCollection serviceCollection, string name, bool kerberos, bool noSsl) => serviceCollection - .AddHttpClient(name) + private static IServiceCollection AddHttpClient(this IServiceCollection serviceCollection, string name, bool noSsl = false) => serviceCollection + .AddHttpClient(name, _ => { }) .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler { - UseDefaultCredentials = kerberos, ServerCertificateCustomValidationCallback = noSsl ? delegate { return true; } : null }) .Services; From a37f4acc594b432c95484acc6771b285c4d05b00 Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 18:08:18 +0000 Subject: [PATCH 65/71] clean up Kerberos from migrate repo --- src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs | 6 ------ .../Commands/MigrateRepo/MigrateRepoCommandArgs.cs | 10 ++-------- .../Commands/MigrateRepo/MigrateRepoCommandHandler.cs | 11 ++++------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index caa0ac2c7..aa5626b52 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -35,7 +35,6 @@ public MigrateRepoCommand() : base( AddOption(AwsRegion); AddOption(QueueOnly); AddOption(TargetRepoVisibility.FromAmong("public", "private", "internal")); - AddOption(Kerberos); AddOption(Verbose); AddOption(KeepArchive); AddOption(NoSslVerify); @@ -119,11 +118,6 @@ public MigrateRepoCommand() : base( name: "--target-repo-visibility", description: "The visibility of the target repo. Defaults to private. Valid values are public, private, or internal."); - public Option Kerberos { get; } = new( - name: "--kerberos", - description: "Use Kerberos authentication for downloading the export archive off of the GitLab server.") - { IsHidden = true }; - public Option Verbose { get; } = new("--verbose"); public Option KeepArchive { get; } = new( diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index 6ff8403dc..220ea9584 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -30,7 +30,6 @@ public class MigrateRepoCommandArgs : CommandArgs public string TargetRepoVisibility { get; set; } public string TargetApiUrl { get; set; } public string TargetUploadsUrl { get; set; } - public bool Kerberos { get; set; } public string GitlabServerUrl { get; set; } public string GitlabGroup { get; set; } @@ -96,11 +95,6 @@ private void ValidateNoGenerateOptions() private void ValidateGenerateOptions() { - if (Kerberos && GitlabPat.HasValue()) - { - throw new OctoshiftCliException("--gitlab-pat cannot be provided with --kerberos."); - } - if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) { throw new OctoshiftCliException("Both --gitlab-group and --gitlab-project must be provided."); @@ -127,12 +121,12 @@ private void ValidateImportOptions() { if (GithubOrg.IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("--github-org must be provided in order to import the Bitbucket archive."); + throw new OctoshiftCliException("--github-org must be provided in order to import the GitLab archive."); } if (GithubRepo.IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("--github-repo must be provided in order to import the Bitbucket archive."); + throw new OctoshiftCliException("--github-repo must be provided in order to import the GitLab archive."); } } } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index a9de56b87..fd99a86c0 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -218,7 +218,7 @@ private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSo archiveUrl ??= args.ArchiveUrl; - var bbsRepoUrl = GetGitlabProjectUrl(args); + var gitlabRepoUrl = GetGitlabProjectUrl(args); args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); @@ -227,7 +227,7 @@ private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSo try { - migrationId = await _githubApi.StartGitlabMigration(migrationSourceId, bbsRepoUrl, githubOrgId, args.GithubRepo, args.GithubPat, archiveUrl, args.TargetRepoVisibility); + migrationId = await _githubApi.StartGitlabMigration(migrationSourceId, gitlabRepoUrl, githubOrgId, args.GithubRepo, args.GithubPat, archiveUrl, args.TargetRepoVisibility); } catch (OctoshiftCliException ex) when (ex.Message == $"A repository called {args.GithubOrg}/{args.GithubRepo} already exists") { @@ -288,12 +288,9 @@ private void ValidateOptions(MigrateRepoCommandArgs args) { if (args.ShouldGenerateArchive()) { - if (!args.Kerberos) + if (GetGitlabPat(args).IsNullOrWhiteSpace()) { - if (GetGitlabPat(args).IsNullOrWhiteSpace()) - { - throw new OctoshiftCliException("BBS password must be either set as BBS_PAT environment variable or passed as --bbs-pat."); - } + throw new OctoshiftCliException("GitLab PAT must be either set as GITLAB_PAT environment variable or passed as --gitlab-pat."); } } From fd616921ca41b275ffcb40c6c0df8ee661b910de Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 18:36:54 +0000 Subject: [PATCH 66/71] clean-up bbs in inventoryreport --- .../InventoryReport/InventoryReportCommandHandler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index a4eebed8c..72240a3ee 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -13,20 +13,20 @@ public class InventoryReportCommandHandler : ICommandHandler Date: Tue, 19 May 2026 20:17:18 +0000 Subject: [PATCH 67/71] fix some existing tests --- .../MigrateRepoCommandArgsTests.cs | 863 +++++------------- .../gl2gh/Factories/GitlabApiFactoryTests.cs | 52 +- .../Services/GitlabInspectorServiceTests.cs | 239 ++--- .../ProjectsCsvGeneratorServiceTests.cs | 155 ++-- 4 files changed, 412 insertions(+), 897 deletions(-) diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs index 500bad660..ce94458a6 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs @@ -4,649 +4,266 @@ using OctoshiftCLI.Services; using Xunit; -namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgsTests { - public class MigrateRepoCommandArgsTests + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + + private const string ARCHIVE_PATH = "path/to/archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_SESSION_TOKEN = "aws-session-token"; + private const string AWS_REGION = "aws-region"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PAT = "gitlab-pat"; + + [Fact] + public void It_Throws_When_Neither_Gitlab_Server_Url_Nor_Archive_Source_Is_Provided() { - private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); - - private const string ARCHIVE_PATH = "path/to/archive.tar"; - private const string ARCHIVE_URL = "https://archive-url/bbs-archive.tar"; - private const string GITHUB_ORG = "target-org"; - private const string GITHUB_REPO = "target-repo"; - private const string GITHUB_PAT = "github pat"; - private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; - private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; - private const string AWS_SESSION_TOKEN = "aws-session-token"; - private const string AWS_REGION = "aws-region"; - private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; - private const string AWS_BUCKET_NAME = "aws-bucket-name"; - private const string BBS_HOST = "our-bbs-server.com"; - private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; - private const string BBS_PROJECT = "bbs-project"; - private const string BBS_REPO = "bbs-repo"; - private const string SSH_USER = "ssh-user"; - private const string PRIVATE_KEY = "private-key"; - private const string SMB_USER = "smb-user"; - private const string SMB_PASSWORD = "smb-password"; - - [Fact] - public void It_Throws_When_Kerberos_Is_Set_And_Gitlab_Password_Is_Provided() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabPassword = BBS_PASSWORD, - Kerberos = true - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-password*--kerberos*"); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Access_Key_Provided() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsAccessKey = AWS_ACCESS_KEY_ID - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*AWS S3*--aws-bucket-name*"); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Provided_With_UseGithubStorage_Option() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsBucketName = AWS_BUCKET_NAME, - UseGithubStorage = true - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--use-github-storage flag was provided with an AWS S3 Bucket name*"); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Provided_With_AzureStorageConnectionString_Option() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsBucketName = AWS_BUCKET_NAME, - UseGithubStorage = true - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*Archive cannot be uploaded to both locations."); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Secret_Key_Provided() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsSecretKey = AWS_SECRET_ACCESS_KEY - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*AWS S3*--aws-bucket-name*"); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Session_Token_Provided() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsSessionToken = AWS_SESSION_TOKEN - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*AWS S3*--aws-bucket-name*"); - } - - [Fact] - public void It_Throws_When_Aws_Bucket_Name_Not_Provided_But_Aws_Region_Provided() - { - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsRegion = AWS_REGION - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*AWS S3*--aws-bucket-name*"); - } - - [Fact] - public void Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Project() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabRepo = BBS_REPO, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-project*"); - } - - [Fact] - public void Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Repo() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-repo*"); - } - - [Fact] - public void It_Throws_When_Kerberos_Is_Set_And_Gitlab_Username_Is_Provided() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabUsername = BBS_USERNAME, - Kerberos = true - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-username*--kerberos*"); - } - - [Fact] - public void Errors_If_Gitlab_Password_Is_Provided_With_Archive_Path() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GitlabPassword = BBS_USERNAME - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-username*--bbs-password*--archive-path*"); - } - - [Fact] - public void Errors_If_Gitlab_Password_Is_Provided_With_Archive_Url() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GitlabPassword = BBS_USERNAME - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-username*--bbs-password*--archive-url*"); - } - - [Fact] - public void Errors_If_No_Ssl_Verify_Is_Provided_With_Archive_Path() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - NoSslVerify = true - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--no-ssl-verify*--archive-path*"); - } - - [Fact] - public void Errors_If_No_Ssl_Verify_Is_Provided_With_Archive_Url() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - NoSslVerify = true - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--no-ssl-verify*--archive-url*"); - } - - [Fact] - public void Errors_If_Ssh_User_Is_Provided_With_Archive_Path() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*SSH*SMB*--archive-path*"); - } - - [Fact] - public void Errors_If_Ssh_User_Is_Provided_With_Archive_Url() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*SSH*SMB*--archive-url*"); - } - - [Fact] - public void Errors_If_Smb_User_Is_Provided_With_Archive_Path() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - SmbUser = SMB_USER, - SmbPassword = SMB_PASSWORD, - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*SSH*SMB*--archive-path*"); - } - - [Fact] - public void Errors_If_Smb_User_Is_Provided_With_Archive_Url() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - SmbUser = SMB_USER, - SmbPassword = SMB_PASSWORD, - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*SSH*SMB*--archive-url*"); - } - - [Fact] - public void It_Throws_If_Github_Org_Is_Provided_But_Github_Repo_Is_Not() - { - // Act - var args = new MigrateRepoCommandArgs - { - GithubOrg = GITHUB_ORG, - GitlabServerUrl = BBS_SERVER_URL, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GithubPat = GITHUB_PAT, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--github-repo*"); - } - - [Fact] - public void It_Throws_If_Archive_Url_Is_Provided_But_Github_Org_Is_Not() + var args = new MigrateRepoCommandArgs { - // Act - var args = new MigrateRepoCommandArgs - { - GithubPat = GITHUB_PAT, - ArchiveUrl = ARCHIVE_URL, - GithubRepo = GITHUB_REPO - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--github-org*"); - } - - [Fact] - public void It_Throws_If_Archive_Url_Is_Provided_But_Github_Repo_Is_Not() - { - // Act - var args = new MigrateRepoCommandArgs - { - GithubPat = GITHUB_PAT, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG - }; - - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--github-repo*"); - } - - [Fact] - public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_User_And_Smb_User_Are_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SmbUser = SMB_USER - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; - [Fact] - public void Errors_When_Archive_Download_Host_Provided_Without_Ssh_Or_Smb_Options() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - ArchiveDownloadHost = "somehost" - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } - - [Fact] - public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_User_And_Smb_Password_Are_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SmbPassword = SMB_PASSWORD - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } - - [Fact] - public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_Private_Key_And_Smb_User_Are_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshPrivateKey = PRIVATE_KEY, - SmbUser = SMB_USER - }; - - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-server-url*--archive-path*--archive-url*"); + } - [Fact] - public void Invoke_With_Gitlab_Server_Url_Throws_When_Both_Ssh_Private_Key_And_Smb_Password_Are_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - SmbPassword = SMB_PASSWORD - }; + [Fact] + public void It_Throws_When_Both_Archive_Path_And_Archive_Url_Are_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--archive-path*--archive-url*"); + } - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } + [Fact] + public void It_Throws_When_Gitlab_Group_Or_Project_Is_Missing_When_Generating_Archive() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-group*--gitlab-project*"); + } - [Fact] - public void Invoke_With_Gitlab_Server_Url_Throws_When_Ssh_User_Is_Provided_And_Ssh_Private_Key_Is_Not_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER - }; + [Fact] + public void It_Throws_When_Gitlab_Pat_Is_Provided_With_Archive_Path() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GitlabPat = GITLAB_PAT + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-pat*--archive-path*--archive-url*"); + } - args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().ThrowExactly(); - } + [Fact] + public void It_Throws_When_No_Ssl_Verify_Is_Provided_With_Archive_Url() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + NoSslVerify = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--archive-path*--archive-url*"); + } - [Fact] - public void Errors_If_Archive_Url_And_Archive_Path_Are_Passed() - { - // Act - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - }; + [Fact] + public void It_Throws_When_Aws_Access_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsAccessKey = AWS_ACCESS_KEY_ID + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--archive-path*--archive-url*"); - } + [Fact] + public void It_Throws_When_Aws_Secret_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSecretKey = AWS_SECRET_ACCESS_KEY + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } - [Fact] - public void Allows_GitlabServer_Url_And_Archive_Url_To_Be_Passed_Together() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - }; + [Fact] + public void It_Throws_When_Aws_Session_Token_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSessionToken = AWS_SESSION_TOKEN + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .NotThrow(); - } + [Fact] + public void It_Throws_When_Aws_Region_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsRegion = AWS_REGION + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } - [Fact] - public void Allows_GitlabServer_Url_And_Archive_Path_To_Be_Passed_Together() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - }; + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Aws_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with an AWS S3 Bucket name*"); + } - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .NotThrow(); - } + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Azure_Storage_Connection_String() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with a connection string*"); + } - [Fact] - public void Errors_If_GitlabServer_Url_Archive_Path_And_Archive_Url_Are_Not_Provided() + [Fact] + public void It_Throws_When_Github_Org_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs { - // Act - var args = new MigrateRepoCommandArgs - { - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - }; + ArchiveUrl = ARCHIVE_URL, + GithubRepo = GITHUB_REPO + }; - // Assert - args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--bbs-server-url*--archive-path*--archive-url*"); - } + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-org*GitLab archive*"); + } - [Fact] - public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() + [Fact] + public void It_Throws_When_Github_Repo_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs { - var args = new MigrateRepoCommandArgs - { - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - SshPort = 7999 - }; + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG + }; - args.Validate(_mockOctoLogger.Object); + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-repo*GitLab archive*"); + } - _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); - } + [Fact] + public void Valid_Generate_And_Upload_Args_Do_Not_Throw() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().NotThrow(); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs index 53309ddb5..6c8b728c1 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs @@ -6,42 +6,25 @@ using OctoshiftCLI.Services; using Xunit; -namespace OctoshiftCLI.Tests.bbs2gh.Factories; +namespace OctoshiftCLI.Tests.GitlabToGithub.Factories; public class GitlabApiFactoryTests { - private const string BBS_SERVER_URL = "http://bbs.contoso.com:7990"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); - private readonly Mock _mockHttpClientFactory = new Mock(); + private readonly Mock _mockHttpClientFactory = new(); private readonly GitlabApiFactory _gitlabApiFactory; public GitlabApiFactoryTests() { - _gitlabApiFactory = new GitlabApiFactory(_mockOctoLogger.Object, _mockHttpClientFactory.Object, _mockEnvironmentVariableProvider.Object, null, null); + _gitlabApiFactory = new GitlabApiFactory(_mockOctoLogger.Object, _mockHttpClientFactory.Object, _mockEnvironmentVariableProvider.Object, null, null, null); } [Fact] - public void Should_Create_GitlabApi_For_Source_Gitlab_Api_With_Kerberos() - { - using var httpClient = new HttpClient(); - - _mockHttpClientFactory - .Setup(x => x.CreateClient("Kerberos")) - .Returns(httpClient); - - // Act - var githubApi = _gitlabApiFactory.CreateKerberos(BBS_SERVER_URL); - - // Assert - githubApi.Should().NotBeNull(); - httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); - } - - [Fact] - public void Should_Create_GitlabApi_For_Source_Gitlab_Api_With_Default() + public void Should_Create_GitlabApi_With_Default() { using var httpClient = new HttpClient(); @@ -50,10 +33,10 @@ public void Should_Create_GitlabApi_For_Source_Gitlab_Api_With_Default() .Returns(httpClient); // Act - var githubApi = _gitlabApiFactory.Create(BBS_SERVER_URL, "user", "pass"); + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat"); // Assert - githubApi.Should().NotBeNull(); + gitlabApi.Should().NotBeNull(); httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); } @@ -67,27 +50,10 @@ public void Should_Create_GitlabApi_With_No_Ssl_Verify() .Returns(httpClient); // Act - var githubApi = _gitlabApiFactory.Create(BBS_SERVER_URL, "user", "pass", true); - - // Assert - githubApi.Should().NotBeNull(); - httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); - } - - [Fact] - public void Should_Create_GitlabApi_With_Kerberos_And_No_Ssl_Verify() - { - using var httpClient = new HttpClient(); - - _mockHttpClientFactory - .Setup(x => x.CreateClient("KerberosNoSSL")) - .Returns(httpClient); - - // Act - var githubApi = _gitlabApiFactory.CreateKerberos(BBS_SERVER_URL, true); + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat", true); // Assert - githubApi.Should().NotBeNull(); + gitlabApi.Should().NotBeNull(); httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs index 1586ff5ea..6df183f3a 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs @@ -1,171 +1,106 @@ -using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Moq; -using Octoshift.Models; using OctoshiftCLI.GitlabToGithub; using OctoshiftCLI.Services; using Xunit; -namespace OctoshiftCLI.Tests.GitlabToGithub.Commands +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class GitlabInspectorServiceTests { - public class GitlabInspectorServiceTests + private readonly OctoLogger _logger = TestHelpers.CreateMock().Object; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly GitlabInspectorService _service; + + private const string GROUP_PATH_1 = "group-1"; + private const string GROUP_NAME_1 = "Group 1"; + private const string GROUP_PATH_2 = "group-2"; + private const string GROUP_NAME_2 = "Group 2"; + + public GitlabInspectorServiceTests() => _service = new(_logger, _mockGitlabApi.Object); + + [Fact] + public async Task GetGroups_Returns_Path_And_Name() { - private readonly OctoLogger _logger = TestHelpers.CreateMock().Object; - private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); - private readonly GitlabInspectorService _service; - - private const string BBS_FOO_PROJECT_KEY = "FP"; - private const string BBS_BAR_PROJECT_KEY = "BP"; - - public GitlabInspectorServiceTests() => _service = new(_logger, _mockGitlabApi.Object); - - [Fact] - public async Task GetProjects_Should_Return_All_Projects() - { - // Arrange - var project1 = "project1"; - var project2 = "project2"; - var projects = new[] { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: project1), - (Id: 1, Key: BBS_BAR_PROJECT_KEY, Name: project2) - }; - - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(projects); - - // Act - var result = await _service.GetProjects(); - - // Assert - result.Should().BeEquivalentTo([(BBS_FOO_PROJECT_KEY, project1), (BBS_BAR_PROJECT_KEY, project2)]); - } - - [Fact] - public async Task GetRepos_Should_Return_All_Repos() - { - // Arrange - var repo1 = "repo1"; - var repo2 = "repo2"; - var repos = new[] - { - (Id: 1, Slug: repo1, Name: repo1), - (Id: 2, Slug: repo2, Name: repo2) - }; - - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); - - // Act - var result = await _service.GetRepos(BBS_FOO_PROJECT_KEY); - - // Assert - result.Should().BeEquivalentTo(new List() { new() { Name = repo1, Slug = repo1 }, new() { Name = repo2, Slug = repo2 } }); - } - - [Fact] - public async Task GetRepoCount_Should_Return_Count() - { - // Arrange - var project = "project"; - var projects = new[] { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: project) - }; - var repo1 = "repo1"; - var repo2 = "repo2"; - var repos = new[] - { - (Id: 1, Slug: repo1, Name: repo1), - (Id: 2, Slug: repo2, Name: repo2) - }; - var expectedCount = 2; - - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(projects); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); - - // Act - var result = await _service.GetRepoCount(); - - // Assert - result.Should().Be(expectedCount); - } - - [Fact] - public async Task GetRepoCount_With_Project_Keys_Should_Return_Count() - { - // Arrange - var repo1 = "repo1"; - var repo2 = "repo2"; - var repos = new[] - { - (Id: 1, Slug: repo1, Name: repo1), - (Id: 2, Slug: repo2, Name: repo2) - }; - var expectedCount = 2; - - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repos); - - // Act - var result = await _service.GetRepoCount(new[] { BBS_FOO_PROJECT_KEY }); - - // Assert - result.Should().Be(expectedCount); - _mockGitlabApi.Verify(m => m.GetProjects(), Times.Never); - } - - [Fact] - public async Task GetPullRequestCount_Should_Return_Count() - { - // Arrange - var project = "project"; - var repo1 = "repo1"; - var repo2 = "repo2"; - var repos = new[] + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] { - (Id: 1, Slug: repo1, Name: repo1), - (Id: 2, Slug: repo2, Name: repo2) - }; + (Id: 1L, Path: GROUP_PATH_1, Name: GROUP_NAME_1), + (Id: 2L, Path: GROUP_PATH_2, Name: GROUP_NAME_2) + }); - var prs1 = new[] + var result = await _service.GetGroups(); + + result.Should().BeEquivalentTo([(GROUP_PATH_1, GROUP_NAME_1), (GROUP_PATH_2, GROUP_NAME_2)]); + } + + [Fact] + public async Task GetProjects_Returns_Projects_For_Group() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] { - (Id: 1, Name: "pr1"), - (Id: 2, Name: "pr2") - }; - var prs2 = new[] + (Id: 1L, Path: "project-1", Name: "Project 1", Archived: false), + (Id: 2L, Path: "project-2", Name: "Project 2", Archived: true) + }); + + var result = (await _service.GetProjects(GROUP_PATH_1)).ToList(); + + result.Should().HaveCount(2); + result[0].Path.Should().Be("project-1"); + result[0].Name.Should().Be("Project 1"); + result[1].Path.Should().Be("project-2"); + result[1].Name.Should().Be("Project 2"); + } + + [Fact] + public async Task GetProjectCount_For_Group_Returns_Count() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] { - (Id: 3, Name: "pr3") - }; - var expectedCount = 3; - - _mockGitlabApi.Setup(m => m.GetRepos(project)).ReturnsAsync(repos); - _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo1)).ReturnsAsync(prs1); - _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo2)).ReturnsAsync(prs2); - - // Act - var result = await _service.GetPullRequestCount(project); - - // Assert - result.Should().Be(expectedCount); - } - - [Fact] - public async Task GetRepositoryPullRequestCount_Should_Return_Count() - { - // Arrange - var project = "project"; - var repo = "repo1"; - var prs = new[] + (Id: 1L, Path: "p1", Name: "p1", Archived: false), + (Id: 2L, Path: "p2", Name: "p2", Archived: false) + }); + + var result = await _service.GetProjectCount(GROUP_PATH_1); + + result.Should().Be(2); + } + + [Fact] + public async Task GetProjectCount_For_Multiple_Groups_Returns_Sum() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] { (Id: 1L, Path: "p1", Name: "p1", Archived: false) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_2)) + .ReturnsAsync(new[] { - (Id: 1, Name: "pr1"), - (Id: 2, Name: "pr2") - }; - var expectedCount = 2; + (Id: 2L, Path: "p2", Name: "p2", Archived: false), + (Id: 3L, Path: "p3", Name: "p3", Archived: false) + }); + + var result = await _service.GetProjectCount(new[] { GROUP_PATH_1, GROUP_PATH_2 }); - _mockGitlabApi.Setup(m => m.GetRepositoryPullRequests(project, repo)).ReturnsAsync(prs); + result.Should().Be(3); + } + + [Fact] + public async Task GetProjectMergeRequestCount_Returns_Count_From_Api() + { + _mockGitlabApi + .Setup(m => m.GetMergeRequestCount(GROUP_PATH_1, "project-1")) + .ReturnsAsync(7); - // Act - var result = await _service.GetRepositoryPullRequestCount(project, repo); + var result = await _service.GetProjectMergeRequestCount(GROUP_PATH_1, "project-1"); - // Assert - result.Should().Be(expectedCount); - } + result.Should().Be(7); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs index 8086eb131..9f799984e 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs @@ -1,92 +1,89 @@ using System; -using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; using Moq; +using Octoshift.Models; using OctoshiftCLI.GitlabToGithub; using OctoshiftCLI.GitlabToGithub.Factories; using OctoshiftCLI.Services; using Xunit; -namespace OctoshiftCLI.Tests.GitlabToGithub.Commands +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class ProjectsCsvGeneratorServiceTests { - public class ProjectsCsvGeneratorServiceTests + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const bool NO_SSL_VERIFY = true; + + private const string GROUP_PATH = "group-1"; + private const string GROUP_NAME = "Group 1"; + private const string PROJECT_PATH = "project-1"; + private const string PROJECT_NAME = "Project 1"; + + private const string FULL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived,mr-count"; + private const string MINIMAL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + + private readonly ProjectsCsvGeneratorService _service; + + public ProjectsCsvGeneratorServiceTests() + { + _mockGitlabApiFactory.Setup(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); + _service = new ProjectsCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); + } + + [Fact] + public async Task Generate_Returns_Csv_For_Single_Group() + { + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + const long repoSize = 1234; + const long attachmentsSize = 5678; + const int mrCount = 7; + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + _mockGitlabInspectorService.Setup(m => m.GetProjectMergeRequestCount(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(mrCount); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH); + + var expected = + $"{FULL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"\",{mrCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Returns_Minimal_Csv_When_Requested() { - private const string FULL_CSV_HEADER = "project-key,project-name,url,repo-count,pr-count"; - private const string MINIMAL_CSV_HEADER = "project-key,project-name,url,repo-count"; - - private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); - - private const string BBS_SERVER_URL = "http://bbs-server-url"; - private const string BBS_FOO_PROJECT = "project1"; - private const string BBS_BAR_PROJECT = "project2"; - private const string BBS_FOO_PROJECT_KEY = "FP"; - private const string BBS_BAR_PROJECT_KEY = "BP"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; - private const bool NO_SSL_VERIFY = true; - private readonly (string, string) _bbsProject = (BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT); - private readonly IEnumerable<(string, string)> _bbsProjects = [(BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT), (BBS_BAR_PROJECT_KEY, BBS_BAR_PROJECT)]; - - private readonly ProjectsCsvGeneratorService _service; - - public ProjectsCsvGeneratorServiceTests() - { - _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); - _service = new ProjectsCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); - } - - [Fact] - public async Task Generate_Should_Return_Correct_Csv_For_One_Project() - { - // Arrange - var repoCount = 82; - var prCount = 822; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProject(BBS_FOO_PROJECT_KEY)).ReturnsAsync(_bbsProject); - _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repoCount); - _mockGitlabInspectorService.Setup(m => m.GetPullRequestCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(prCount); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_FOO_PROJECT_KEY); - - // Assert - var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}"; - - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}\",{repoCount},{prCount}{Environment.NewLine}"; - - result.Should().Be(expected); - } - - [Fact] - public async Task Generate_Should_Return_Minimal_Csv_When_Minimal_Is_True() - { - // Arrange - const int repoCount1 = 82; - const int repoCount2 = 0; - const bool minimal = true; - - _mockGitlabApiFactory.Setup(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); - - _mockGitlabInspectorService.Setup(m => m.GetProjects()).ReturnsAsync(_bbsProjects); - _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_FOO_PROJECT_KEY)).ReturnsAsync(repoCount1); - _mockGitlabInspectorService.Setup(m => m.GetRepoCount(BBS_BAR_PROJECT_KEY)).ReturnsAsync(repoCount2); - - // Act - var result = await _service.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, "", minimal); - - // Assert - var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}"; - expected += $"\"{BBS_FOO_PROJECT_KEY}\",\"{BBS_FOO_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_FOO_PROJECT_KEY}\",{repoCount1}{Environment.NewLine}"; - expected += $"\"{BBS_BAR_PROJECT_KEY}\",\"{BBS_BAR_PROJECT}\",\"{BBS_SERVER_URL.TrimEnd('/')}/projects/{BBS_BAR_PROJECT_KEY}\",{repoCount2}{Environment.NewLine}"; - - result.Should().Be(expected); - _mockGitlabInspectorService.Verify(m => m.GetPullRequestCount(It.IsAny()), Times.Never); - } + const long repoSize = 1234; + const long attachmentsSize = 5678; + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH, minimal: true); + + var expected = + $"{MINIMAL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetProjectMergeRequestCount(It.IsAny(), It.IsAny()), Times.Never); } } From 8c6a8c898fc59fb952ba735c1c9a514d849e177f Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 20:37:08 +0000 Subject: [PATCH 68/71] update generate script command --- .../GenerateScriptCommandArgsTests.cs | 24 - .../GenerateScriptCommandHandlerTests.cs | 847 +++--------------- .../GenerateScriptCommandTests.cs | 84 +- .../GenerateScript/GenerateScriptCommand.cs | 45 +- .../GenerateScriptCommandArgs.cs | 21 +- .../GenerateScriptCommandHandler.cs | 17 +- 6 files changed, 157 insertions(+), 881 deletions(-) diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs index 717bb0b17..9e617347e 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -11,30 +11,6 @@ public class GenerateScriptCommandArgsTests private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); private readonly GenerateScriptCommandArgs _args = new(); - [Fact] - public void It_Throws_If_GitlabServer_Url_Is_Not_Provided_But_No_Ssl_Verify_Is_Provided() - { - // Act - _args.NoSslVerify = true; - _args.GitlabServerUrl = ""; - - // Assert - _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) - .Should() - .ThrowExactly() - .WithMessage("*--no-ssl-verify*--bbs-server-url*"); - } - - [Fact] - public void Invoke_With_Ssh_Port_Set_To_7999_Logs_Warning() - { - _args.SshPort = 7999; - - _args.Validate(_mockOctoLogger.Object); - - _mockOctoLogger.Verify(x => x.LogWarning(It.Is(x => x.ToLower().Contains("--ssh-port is set to 7999")))); - } - [Fact] public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() { diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs index 735758eaf..ffabddc50 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -1,11 +1,10 @@ -using System; using System.IO; using System.Linq; using System.Threading.Tasks; +using FluentAssertions; using Moq; -using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; using OctoshiftCLI.Contracts; -using OctoshiftCLI.Extensions; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; using OctoshiftCLI.Services; using Xunit; @@ -22,32 +21,18 @@ public class GenerateScriptCommandHandlerTests private readonly GenerateScriptCommandHandler _handler; private const string GITHUB_ORG = "GITHUB-ORG"; - private const string BBS_SERVER_URL = "http://bbs-server-url"; - private const string BBS_USERNAME = "BBS-USERNAME"; - private const string BBS_PASSWORD = "BBS-PASSWORD"; - private const string SSH_USER = "SSH-USER"; - private const string SSH_PRIVATE_KEY = "path-to-ssh-private-key"; - private const string ARCHIVE_DOWNLOAD_HOST = "archive-download-host"; - private const int SSH_PORT = 2211; - private const string SMB_USER = "SMB-USER"; - private const string SMB_DOMAIN = "SMB-DOMAIN"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; private const string OUTPUT = "unit-test-output"; - private const string BBS_FOO_PROJECT_KEY = "FP"; - private const string BBS_FOO_PROJECT_NAME = "BBS-FOO-PROJECT-NAME"; - private const string BBS_BAR_PROJECT_KEY = "BBS-BAR-PROJECT-NAME"; - private const string BBS_BAR_PROJECT_NAME = "BP"; - private const string BBS_FOO_REPO_1_SLUG = "foorepo1"; - private const string BBS_FOO_REPO_1_NAME = "BBS-FOO-REPO-1-NAME"; - private const string BBS_FOO_REPO_2_SLUG = "foorepo2"; - private const string BBS_FOO_REPO_2_NAME = "BBS-FOO-REPO-2-NAME"; - private const string BBS_BAR_REPO_1_SLUG = "barrepo1"; - private const string BBS_BAR_REPO_1_NAME = "BBS-BAR-REPO-1-NAME"; - private const string BBS_BAR_REPO_2_SLUG = "barrepo2"; - private const string BBS_BAR_REPO_2_NAME = "BBS-BAR-REPO-2-NAME"; - private const string BBS_SHARED_HOME = "BBS-SHARED-HOME"; + private const string GROUP_PATH_FOO = "group-foo"; + private const string GROUP_NAME_FOO = "Group Foo"; + private const string GROUP_PATH_BAR = "group-bar"; + private const string GROUP_NAME_BAR = "Group Bar"; + private const string PROJECT_PATH_1 = "project-1"; + private const string PROJECT_NAME_1 = "Project 1"; + private const string PROJECT_PATH_2 = "project-2"; + private const string PROJECT_NAME_2 = "Project 2"; private const string AWS_BUCKET_NAME = "AWS-BUCKET-NAME"; - private const string AWS_REGION = "AWS_REGION"; - private const string UPLOADS_URL = "UPLOADS-URL"; + private const string AWS_REGION = "us-east-1"; public GenerateScriptCommandHandlerTests() { @@ -57,783 +42,167 @@ public GenerateScriptCommandHandlerTests() _mockFileSystemProvider.Object, _mockGitlabApi.Object, _mockEnvironmentVariableProvider.Object); - - _mockEnvironmentVariableProvider.Setup(m => m.GitlabUsername(It.IsAny())).Returns(BBS_USERNAME); - _mockEnvironmentVariableProvider.Setup(m => m.GitlabPassword(It.IsAny())).Returns(BBS_PASSWORD); - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] { (1, BBS_FOO_PROJECT_KEY, BBS_FOO_PROJECT_NAME) }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] { (1, BBS_FOO_REPO_1_SLUG, BBS_FOO_REPO_1_NAME) }); - } - - [Fact] - public async Task No_Projects() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT) - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 33, 0) == ""))); - } - - [Fact] - public async Task Validates_Env_Vars() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT), - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:GH_PAT) { - Write-Error ""GH_PAT environment variable must be set to a valid GitHub Personal Access Token with the appropriate scopes. For more information see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#creating-a-personal-access-token-for-github-enterprise-importer"" - exit 1 -} else { - Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" -} - -if (-not $env:BBS_PASSWORD) { - Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" - exit 1 -} else { - Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" -} - -if (-not $env:BBS_USERNAME) { - Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call Bitbucket Server/Data Center API's to generate a migration archive."" - exit 1 -} else { - Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to Bitbucket Server/Data Center APIs."" -} - -if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { - Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" - exit 1 -} else { - Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 9, 0) == TrimNonExecutableLines(expected, 0, 0)))); - } - - [Fact] - public async Task Validates_Env_Vars_BBS_USERNAME_Not_Validated_When_Passed_As_Arg() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT), - GitlabUsername = BBS_USERNAME, - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:BBS_USERNAME) { - Write-Error ""BBS_USERNAME environment variable must be set to a valid user that will be used to call BBS API's to generate a migration archive."" - exit 1 -} else { - Write-Host ""BBS_USERNAME environment variable is set and will be used to authenticate to BBS APIs."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); - } - - [Fact] - public async Task Validates_Env_Vars_BBS_PASSWORD_Not_Validated_When_Kerberos() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT), - Kerberos = true, - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:BBS_PASSWORD) { - Write-Error ""BBS_PASSWORD environment variable must be set to a valid password that will be used to call BBS API's to generate a migration archive."" - exit 1 -} else { - Write-Host ""BBS_PASSWORD environment variable is set and will be used to authenticate to BBS APIs."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); - } - - [Fact] - public async Task Validates_Env_Vars_AWS() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - AwsBucketName = AWS_BUCKET_NAME, - Output = new FileInfo(OUTPUT), - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:AWS_ACCESS_KEY_ID) { - Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" - exit 1 -} else { - Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" -} -if (-not $env:AWS_SECRET_ACCESS_KEY) { - Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" - exit 1 -} else { - Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); - } - - [Fact] - public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_Not_Validated_When_Aws() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT), - AwsBucketName = AWS_BUCKET_NAME, - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { - Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" - exit 1 -} else { - Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); - } - - [Fact] - public async Task Validates_Env_Vars_AZURE_STORAGE_CONNECTION_STRING_And_AWS_Not_Validated_When_UseGithubStorage() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT), - UseGithubStorage = true - }; - await _handler.Handle(args); - - var expectedAws = @" -if (-not $env:AWS_ACCESS_KEY_ID) { - Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" - exit 1 -} else { - Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" -} -if (-not $env:AWS_SECRET_ACCESS_KEY) { - Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" - exit 1 -} else { - Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" -}"; - - var expectedAzure = @" -if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { - Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" - exit 1 -} else { - Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAws, 0, 0))))); - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => !TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expectedAzure, 0, 0))))); } [Fact] - public async Task Validates_Env_Vars_SMB_PASSWORD() + public async Task No_Output_Path_Does_Not_Write_File() { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(Enumerable.Empty<(int Id, string Key, string Name)>()); + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - Output = new FileInfo(OUTPUT), - SmbUser = SMB_USER, - }; - await _handler.Handle(args); - - var expected = @" -if (-not $env:SMB_PASSWORD) { - Write-Error ""SMB_PASSWORD environment variable must be set to a valid password that will be used to download the migration archive from your BBS server using SMB."" - exit 1 -} else { - Write-Host ""SMB_PASSWORD environment variable is set and will be used to download the migration archive from your BBS server using SMB."" -}"; - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 0, 0).Contains(TrimNonExecutableLines(expected, 0, 0))))); - } - - [Fact] - public async Task No_Repos() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(Enumerable.Empty<(int Id, string Slug, string Name)>()); - - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT) - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => TrimNonExecutableLines(script, 33, 0) == ""))); - } - - [Fact] - public async Task Two_Projects_Two_Repos_Each_All_Options() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - (Id: 2, Key: BBS_BAR_PROJECT_KEY, Name: BBS_BAR_PROJECT_NAME) - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - (Id: 2, Slug: BBS_FOO_REPO_2_SLUG, Name: BBS_FOO_REPO_2_NAME) - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_BAR_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 3, Slug: BBS_BAR_REPO_1_SLUG, Name: BBS_BAR_REPO_1_NAME), - (Id: 4, Slug: BBS_BAR_REPO_2_SLUG, Name: BBS_BAR_REPO_2_NAME) - }); - - var migrateRepoCommand1 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; - var migrateRepoCommand2 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_2_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; - var migrateRepoCommand3 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_BAR_PROJECT_KEY}\" --bbs-repo \"{BBS_BAR_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_BAR_PROJECT_KEY}-{BBS_BAR_REPO_1_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; - var migrateRepoCommand4 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_BAR_PROJECT_KEY}\" --bbs-repo \"{BBS_BAR_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_BAR_PROJECT_KEY}-{BBS_BAR_REPO_2_SLUG}\" --verbose --keep-archive --target-repo-visibility private }}"; - - // Act var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - Output = new FileInfo(OUTPUT), - Verbose = true, - KeepArchive = true + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG }; - await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand1)))); - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand2)))); - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand3)))); - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand4)))); - - _mockEnvironmentVariableProvider.Verify(m => m.GitlabUsername(It.IsAny()), Times.Never); - _mockEnvironmentVariableProvider.Verify(m => m.GitlabPassword(It.IsAny()), Times.Never); - } - - [Fact] - public async Task Filters_By_Project() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - (Id: 2, Key: BBS_BAR_PROJECT_KEY, Name: BBS_BAR_PROJECT_NAME) - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - (Id: 2, Slug: BBS_FOO_REPO_2_SLUG, Name: BBS_FOO_REPO_2_NAME) - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_BAR_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 3, Slug: BBS_BAR_REPO_1_SLUG, Name: BBS_BAR_REPO_1_NAME), - (Id: 4, Slug: BBS_BAR_REPO_2_SLUG, Name: BBS_BAR_REPO_2_NAME) - }); - - var migrateRepoCommand1 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; - var migrateRepoCommand2 = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_2_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_2_SLUG}\" --verbose --target-repo-visibility private }}"; - - // Act - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_FOO_PROJECT_KEY, - GitlabSharedHome = BBS_SHARED_HOME, - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - Output = new FileInfo(OUTPUT), - Verbose = true - }; await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand1)))); - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand2)))); - - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(BBS_BAR_PROJECT_KEY))), Times.Never); + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task One_Repo_With_Kerberos() + public async Task No_Groups_Generates_Header_Only() { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --kerberos --target-repo-visibility private }}"; + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); - // Act var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, + GitlabServerUrl = GITLAB_SERVER_URL, GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - Output = new FileInfo(OUTPUT), - Verbose = true, - Kerberos = true, + Output = new FileInfo(OUTPUT) }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); - } - - [Fact] - public async Task One_Repo_With_No_Ssl_Verify() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --no-ssl-verify --target-repo-visibility private }}"; - - // Act - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - Output = new FileInfo(OUTPUT), - Verbose = true, - NoSslVerify = true, - }; await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain("VALIDATE_GH_PAT".Replace("VALIDATE_GH_PAT", "GH_PAT")); + capturedScript.Should().Contain("GITLAB_PAT"); + capturedScript.Should().NotContain("# =========== Group:"); } [Fact] - public async Task One_Repo_With_Smb() + public async Task Default_Generates_Migrate_Repo_Command_For_Each_Project() { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); - - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; - - // Act - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - SmbUser = SMB_USER, - SmbDomain = SMB_DOMAIN, - Output = new FileInfo(OUTPUT), - Verbose = true - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); - } + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] + { + (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO), + (Id: 2L, Path: GROUP_PATH_BAR, Name: GROUP_NAME_BAR) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false), + (Id: 2L, Path: PROJECT_PATH_2, Name: PROJECT_NAME_2, Archived: false) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_BAR)) + .ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name, bool Archived)>()); - [Fact] - public async Task One_Repo_With_Smb_And_TargetApiUrl() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); - var targetApiUrl = "https://foo.com/api/v3"; - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --target-api-url \"{targetApiUrl}\" --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); - // Act var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, + GitlabServerUrl = GITLAB_SERVER_URL, GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - SmbUser = SMB_USER, - SmbDomain = SMB_DOMAIN, - Output = new FileInfo(OUTPUT), - Verbose = true, - TargetApiUrl = targetApiUrl - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); - } - - [Fact] - public async Task One_Repo_With_Smb_And_Archive_Download_Host() - { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); - - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" --bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" --smb-user \"{SMB_USER}\" --smb-domain {SMB_DOMAIN} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" --github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --target-repo-visibility private }}"; - - // Act - var args = new GenerateScriptCommandArgs - { - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - SmbUser = SMB_USER, - SmbDomain = SMB_DOMAIN, - Output = new FileInfo(OUTPUT), - Verbose = true - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); - } - - [Fact] - public async Task Generated_Script_Contains_The_Cli_Version_Comment() - { - // Arrange - _mockVersionProvider.Setup(m => m.GetCurrentVersion()).Returns("1.1.1"); - const string cliVersionComment = "# =========== Created with CLI version 1.1.1 ==========="; - - // Act - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT) - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(cliVersionComment)))); - } - - [Fact] - public async Task Generated_Script_StartsWith_Shebang() - { - // Arrange - const string shebang = "#!/usr/bin/env pwsh"; - - // Act - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, Output = new FileInfo(OUTPUT) }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.StartsWith(shebang)))); - } - - [Fact] - public async Task Generated_Script_Contains_Exec_Function_Block() - { - // Arrange - const string execFunctionBlock = @" -function Exec { - param ( - [scriptblock]$ScriptBlock - ) - & @ScriptBlock - if ($lastexitcode -ne 0) { - exit $lastexitcode - } -}"; - var args = new GenerateScriptCommandArgs() - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - Output = new FileInfo(OUTPUT) - }; await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(execFunctionBlock)))); + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_FOO} ==========="); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_BAR} ==========="); + capturedScript.Should().Contain($"--gitlab-server-url \"{GITLAB_SERVER_URL}\""); + capturedScript.Should().Contain($"--gitlab-group \"{GROUP_PATH_FOO}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_1}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_2}\""); + capturedScript.Should().Contain($"--github-org \"{GITHUB_ORG}\""); + capturedScript.Should().Contain($"--github-repo \"{GROUP_PATH_FOO}-{PROJECT_PATH_1}\""); + capturedScript.Should().Contain("--target-repo-visibility private"); + capturedScript.Should().Contain("Skipping this group because it has no projects."); + capturedScript.Should().NotContain("--queue-only"); + capturedScript.Should().NotContain("--verbose"); + capturedScript.Should().NotContain("--aws-bucket-name"); + capturedScript.Should().NotContain("--aws-region"); + capturedScript.Should().NotContain("--keep-archive"); + capturedScript.Should().NotContain("--use-github-storage"); + capturedScript.Should().NotContain("--no-ssl-verify"); + capturedScript.Should().NotContain("--kerberos"); } [Fact] - public async Task One_Repo_With_Aws_Bucket_Name_And_Region() + public async Task Includes_Optional_Flags_When_Set() { - // Arrange - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_FOO_PROJECT_KEY, Name: BBS_FOO_PROJECT_NAME), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_FOO_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_FOO_REPO_1_SLUG, Name: BBS_FOO_REPO_1_NAME), - }); + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] { (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] { (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false) }); - var migrateRepoCommand = $"Exec {{ gh bbs2gh migrate-repo --bbs-server-url \"{BBS_SERVER_URL}\" --bbs-username \"{BBS_USERNAME}\" " + - $"--bbs-shared-home \"{BBS_SHARED_HOME}\" --bbs-project \"{BBS_FOO_PROJECT_KEY}\" --bbs-repo \"{BBS_FOO_REPO_1_SLUG}\" " + - $"--ssh-user \"{SSH_USER}\" --ssh-private-key \"{SSH_PRIVATE_KEY}\" --ssh-port {SSH_PORT} --archive-download-host {ARCHIVE_DOWNLOAD_HOST} --github-org \"{GITHUB_ORG}\" " + - $"--github-repo \"{BBS_FOO_PROJECT_KEY}-{BBS_FOO_REPO_1_SLUG}\" --verbose --aws-bucket-name \"{AWS_BUCKET_NAME}\" " + - $"--aws-region \"{AWS_REGION}\" --target-repo-visibility private }}"; + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); - // Act var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, + GitlabServerUrl = GITLAB_SERVER_URL, GithubOrg = GITHUB_ORG, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabSharedHome = BBS_SHARED_HOME, - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, Output = new FileInfo(OUTPUT), Verbose = true, AwsBucketName = AWS_BUCKET_NAME, - AwsRegion = AWS_REGION + AwsRegion = AWS_REGION, + KeepArchive = true }; + await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => script.Contains(migrateRepoCommand)))); + capturedScript.Should().Contain("--verbose"); + capturedScript.Should().Contain($"--aws-bucket-name \"{AWS_BUCKET_NAME}\""); + capturedScript.Should().Contain($"--aws-region \"{AWS_REGION}\""); + capturedScript.Should().Contain("--keep-archive"); + capturedScript.Should().Contain("VALIDATE_AWS_ACCESS_KEY_ID".Replace("VALIDATE_", "").Replace("ID", "ID")); + capturedScript.Should().Contain("AWS_SECRET_ACCESS_KEY"); + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); } [Fact] - public async Task BBS_Single_Repo_With_UseGithubStorage() + public async Task UseGithubStorage_Skips_Azure_And_Aws_Validation() { - // Arrange - var TARGET_API_URL = "https://foo.com/api/v3"; - const string BBS_PROJECT_KEY = "BBS-PROJECT"; - const string BBS_REPO_SLUG = "repo-slug"; - - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), - }); + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); - // Act var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, + GitlabServerUrl = GITLAB_SERVER_URL, GithubOrg = GITHUB_ORG, - Output = new FileInfo("unit-test-output"), - UseGithubStorage = true, - TargetApiUrl = TARGET_API_URL, - GitlabProject = BBS_PROJECT_KEY, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => - script.Contains("--bbs-server-url \"http://bbs-server-url\"") && - script.Contains("--bbs-project \"BBS-PROJECT\"") && - script.Contains("--github-org \"GITHUB-ORG\"") && - script.Contains("--use-github-storage") -))); - } - [Fact] - public async Task BBS_Single_Repo_With_TargetUploadsUrl() - { - // Arrange - var TARGET_API_URL = "https://foo.com/api/v3"; - const string BBS_PROJECT_KEY = "BBS-PROJECT"; - const string BBS_REPO_SLUG = "repo-slug"; - - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] - { - (Id: 1, Key: BBS_PROJECT_KEY, Name: "BBS Project Name"), - }); - _mockGitlabApi.Setup(m => m.GetRepos(BBS_PROJECT_KEY)).ReturnsAsync(new[] - { - (Id: 1, Slug: BBS_REPO_SLUG, Name: "RepoName"), - }); - - // Act - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GithubOrg = GITHUB_ORG, - Output = new FileInfo("unit-test-output"), - TargetApiUrl = TARGET_API_URL, - TargetUploadsUrl = UPLOADS_URL, - GitlabProject = BBS_PROJECT_KEY, - }; await _handler.Handle(args); - // Assert - _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.Is(script => - script.Contains("--bbs-server-url \"http://bbs-server-url\"") && - script.Contains("--bbs-project \"BBS-PROJECT\"") && - script.Contains("--github-org \"GITHUB-ORG\"") && - script.Contains("--target-uploads-url \"UPLOADS-URL\"") - ))); - } - - private string TrimNonExecutableLines(string script, int skipFirst = 9, int skipLast = 0) - { - var lines = script.Split(new[] { Environment.NewLine, "\n" }, StringSplitOptions.RemoveEmptyEntries).AsEnumerable(); - - lines = lines - .Where(x => x.HasValue()) - .Where(x => !x.Trim().StartsWith("#")) - .Skip(skipFirst) - .SkipLast(skipLast); - - var result = string.Join(Environment.NewLine, lines); - return result; + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); + capturedScript.Should().NotContain("AWS_ACCESS_KEY_ID"); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs index 788531326..231f7e2eb 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -11,7 +11,8 @@ namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; public class GenerateScriptCommandTests { - private const string BBS_SERVER_URL = "http://bbs.contoso.com:7990"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; private readonly Mock _mockServiceProvider = new(); private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); @@ -36,94 +37,35 @@ public void Should_Have_Options() { _command.Should().NotBeNull(); _command.Name.Should().Be("generate-script"); - _command.Options.Count.Should().Be(21); + _command.Options.Count.Should().Be(14); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-username", false); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-password", false); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-project", false); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-shared-home", false); - TestHelpers.VerifyCommandOption(_command.Options, "archive-download-host", false); - TestHelpers.VerifyCommandOption(_command.Options, "ssh-user", false); - TestHelpers.VerifyCommandOption(_command.Options, "ssh-private-key", false); - TestHelpers.VerifyCommandOption(_command.Options, "ssh-port", false); - TestHelpers.VerifyCommandOption(_command.Options, "smb-user", false); - TestHelpers.VerifyCommandOption(_command.Options, "smb-domain", false); + TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-project", false); TestHelpers.VerifyCommandOption(_command.Options, "output", false); - TestHelpers.VerifyCommandOption(_command.Options, "kerberos", false, true); TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); TestHelpers.VerifyCommandOption(_command.Options, "aws-bucket-name", false); TestHelpers.VerifyCommandOption(_command.Options, "aws-region", false); TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); - TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); } [Fact] - public void It_Gets_A_Kerberos_HttpClient_When_Kerberos_Is_True() - { - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - Kerberos = true - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, false)); - } - - [Fact] - public void It_Gets_A_Kerberos_With_No_Ssl_Verify_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_True() - { - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - Kerberos = true, - NoSslVerify = true - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, true)); - } - - [Fact] - public void It_Gets_A_Default_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_Not_Set() + public void It_Creates_The_GitlabApi_With_The_Provided_Server_Url_And_Pat() { - var bbsTestUser = "user"; - var bbsTestPassword = "password"; - - var args = new GenerateScriptCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = bbsTestUser, - GitlabPassword = bbsTestPassword - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, false)); - } - - [Fact] - public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_Set() - { - var bbsTestUser = "user"; - var bbsTestPassword = "password"; - var args = new GenerateScriptCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = bbsTestUser, - GitlabPassword = bbsTestPassword, - NoSslVerify = true + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT }; _command.BuildHandler(args, _mockServiceProvider.Object); - _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, bbsTestUser, bbsTestPassword, true)); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); } } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs index 2b1b8d61d..79820f36a 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -18,11 +18,11 @@ public GenerateScriptCommand() : base( AddOption(GitlabServerUrl); AddOption(GithubOrg); AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); AddOption(GitlabPat); + AddOption(GitlabGroup); AddOption(GitlabProject); - AddOption(ArchiveDownloadHost); AddOption(Output); - AddOption(Kerberos); AddOption(Verbose); AddOption(AwsBucketName); AddOption(AwsRegion); @@ -32,23 +32,23 @@ public GenerateScriptCommand() : base( } public Option GitlabServerUrl { get; } = new( - name: "--bbs-server-url", - description: "The full URL of the Bitbucket Server/Data Center to migrate from.") + name: "--gitlab-server-url", + description: "The full URL of the GitLab server to migrate from, e.g. https://gitlab.mycompany.com") { IsRequired = true }; public Option GitlabPat { get; } = new( - name: "--bbs-pat", - description: "The Bitbucket PAT of a user with site admin privileges to get the list of all projects and their repos. If not set will be read from BBS_PASSWORD environment variable." + + name: "--gitlab-pat", + description: "The GitLab PAT of a user with admin privileges to get the list of all groups and their projects. If not set will be read from GITLAB_PAT environment variable." + $"{Environment.NewLine}" + "Note: The PAT will not get included in the generated script and it has to be set as an env variable before running the script."); - public Option GitlabProject { get; } = new( - name: "--bbs-project", - description: "The Bitbucket project to migrate. If not set will migrate all projects."); + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group to migrate. If not set will migrate all groups the user has access to."); - public Option ArchiveDownloadHost { get; } = new( - name: "--archive-download-host", - description: "The host to use to connect to the Bitbucket Server/Data Center instance via SSH or SMB. Defaults to the host from the Bitbucket Server URL (--bbs-server-url)."); + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate. Requires --gitlab-group. If not set will migrate all projects in the group."); public Option GithubOrg { get; } = new("--github-org") { IsRequired = true }; @@ -57,14 +57,9 @@ public GenerateScriptCommand() : base( name: "--output", getDefaultValue: () => new FileInfo("./migrate.ps1")); - public Option Kerberos { get; } = new( - name: "--kerberos", - description: "Use Kerberos authentication for Bitbucket Server.") - { IsHidden = true }; - public Option AwsBucketName { get; } = new( name: "--aws-bucket-name", - description: "If using AWS, the name of the S3 bucket to upload the BBS archive to."); + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); public Option AwsRegion { get; } = new( name: "--aws-region", @@ -77,15 +72,21 @@ public GenerateScriptCommand() : base( name: "--keep-archive", description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); - public Option NoSslVerify { get; } = new( - name: "--no-ssl-verify", - description: "Disables SSL verification when communicating with your Bitbucket Server/Data Center instance. All other migration steps will continue to verify SSL. " + - "If your Bitbucket instance has a self-signed SSL certificate then setting this flag will allow the migration archive to be exported."); public Option TargetApiUrl { get; } = new("--target-api-url") { Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" }; + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); + public Option UseGithubStorage { get; } = new("--use-github-storage") { IsHidden = true, diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs index 80f842024..c03cbdb6f 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -9,32 +9,24 @@ public class GenerateScriptCommandArgs : CommandArgs { public string GitlabServerUrl { get; set; } public string GithubOrg { get; set; } - public string GitlabUsername { get; set; } [Secret] public string GitlabPat { get; set; } public string GitlabGroup { get; set; } - public string GitlabSharedHome { get; set; } - public string ArchiveDownloadHost { get; set; } - public string SshUser { get; set; } - public string SshPrivateKey { get; set; } - public int SshPort { get; set; } - public string SmbUser { get; set; } - public string SmbDomain { get; set; } + public string GitlabProject { get; set; } + public bool NoSslVerify { get; set; } public FileInfo Output { get; set; } - public bool Kerberos { get; set; } public string AwsBucketName { get; set; } public string AwsRegion { get; set; } public bool KeepArchive { get; set; } - public bool NoSslVerify { get; set; } public string TargetApiUrl { get; set; } public string TargetUploadsUrl { get; set; } public bool UseGithubStorage { get; set; } public override void Validate(OctoLogger log) { - if (NoSslVerify && GitlabServerUrl.IsNullOrWhiteSpace()) + if (GitlabProject.HasValue() && GitlabGroup.IsNullOrWhiteSpace()) { - throw new OctoshiftCliException("--no-ssl-verify can only be provided with --bbs-server-url."); + throw new OctoshiftCliException("--gitlab-group must be provided when --gitlab-project is specified."); } if (AwsBucketName.HasValue() && UseGithubStorage) @@ -46,10 +38,5 @@ public override void Validate(OctoLogger log) { throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); } - - if (SshPort == 7999) - { - log?.LogWarning("--ssh-port is set to 7999, which is the default port that Bitbucket Server and Bitbucket Data Center use for Git operations over SSH. This is probably the wrong value, because --ssh-port should be configured with the SSH port used to manage the server where Bitbucket Server/Bitbucket Data Center is running, not the port used for Git operations over SSH."); - } } } diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 2c60d8881..574753cd4 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -57,10 +57,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) content.AppendLine(EXEC_FUNCTION_BLOCK); content.AppendLine(VALIDATE_GH_PAT); - if (!args.Kerberos) - { - content.AppendLine(VALIDATE_GITLAB_PAT); - } + content.AppendLine(VALIDATE_GITLAB_PAT); if (args.AwsBucketName.HasValue() || args.AwsRegion.HasValue()) { content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); @@ -72,7 +69,7 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) } var groups = args.GitlabGroup.HasValue() - ? [args.GitlabGroup] + ? new[] { args.GitlabGroup } : (await _gitlabApi.GetGroups()).Select(x => x.Path); foreach (var groupPath in groups) @@ -84,6 +81,11 @@ private async Task GenerateScript(GenerateScriptCommandArgs args) var projects = await _gitlabApi.GetProjects(groupPath); + if (args.GitlabProject.HasValue()) + { + projects = projects.Where(p => p.Path == args.GitlabProject).ToArray(); + } + if (!projects.Any()) { content.AppendLine("# Skipping this group because it has no projects."); @@ -111,19 +113,18 @@ private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string gi var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; var githubRepoOption = $" --github-repo \"{GetGithubRepoName(gitlabGroup, gitlabProject)}\""; var waitOption = wait ? "" : " --queue-only"; - var kerberosOption = args.Kerberos ? " --kerberos" : ""; var verboseOption = args.Verbose ? " --verbose" : ""; var awsBucketNameOption = args.AwsBucketName.HasValue() ? $" --aws-bucket-name \"{args.AwsBucketName}\"" : ""; var awsRegionOption = args.AwsRegion.HasValue() ? $" --aws-region \"{args.AwsRegion}\"" : ""; var keepArchive = args.KeepArchive ? " --keep-archive" : ""; - var noSslVerify = args.NoSslVerify ? " --no-ssl-verify" : ""; + var noSslVerifyOption = args.NoSslVerify ? " --no-ssl-verify" : ""; var targetRepoVisibility = " --target-repo-visibility private"; var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{gitlabServerUrlOption}{gitlabGroupOption}{gitlabProjectOption}" + - $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{kerberosOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerify}{targetRepoVisibility}{githubStorageOption}"; + $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerifyOption}{targetRepoVisibility}{githubStorageOption}"; } private string Exec(string script) => Wrap(script, "Exec"); From a5359a95e1540d96f4a82301b09e21e131779fde Mon Sep 17 00:00:00 2001 From: Briana J Date: Tue, 19 May 2026 22:33:09 +0000 Subject: [PATCH 69/71] get migrate repo tests passing --- .../MigrateRepoCommandHandlerTests.cs | 1317 ++++------------- .../MigrateRepo/MigrateRepoCommandTests.cs | 247 +--- .../MigrateRepo/MigrateRepoCommand.cs | 4 +- .../MigrateRepo/MigrateRepoCommandArgs.cs | 2 +- .../MigrateRepo/MigrateRepoCommandHandler.cs | 6 +- 5 files changed, 311 insertions(+), 1265 deletions(-) diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs index 74ef78b0b..399b8d114 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs @@ -1,1079 +1,310 @@ using System; using System.IO; -using System.Text; using System.Threading.Tasks; using FluentAssertions; using Moq; using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; -using OctoshiftCLI.GitlabToGithub.Services; -using OctoshiftCLI.Extensions; using OctoshiftCLI.Services; using Xunit; -namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandlerTests { - public class MigrateRepoCommandHandlerTests + private readonly Mock _mockGithubApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApi = TestHelpers.CreateMock(); + private readonly Mock _mockAwsApi = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockHttpDownloadService = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + + private readonly WarningsCountLogger _warningsCountLogger; + private readonly MigrateRepoCommandHandler _handler; + + private const string ARCHIVE_PATH = "/tmp/gitlab-archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string GITHUB_PAT = "github-pat"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_REGION = "eu-west-1"; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PROJECT_URL = $"{GITLAB_SERVER_URL}/{GITLAB_GROUP}/{GITLAB_PROJECT}"; + private const string UNUSED_REPO_URL = "https://not-used"; + + private const string GITHUB_ORG_ID = "github-org-id"; + private const string MIGRATION_SOURCE_ID = "migration-source-id"; + private const string MIGRATION_ID = "migration-id"; + + public MigrateRepoCommandHandlerTests() { - private readonly Mock _mockGithubApi = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); - private readonly Mock _mockAzureApi = TestHelpers.CreateMock(); - private readonly Mock _mockAwsApi = TestHelpers.CreateMock(); - private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); - private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabArchiveDownloader = new(); - private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); - - private readonly WarningsCountLogger _warningsCountLogger; - private readonly MigrateRepoCommandHandler _handler; - - private const string ARCHIVE_PATH = "path/to/archive.tar"; - private const string ARCHIVE_URL = "https://archive-url/bbs-archive.tar"; - private readonly byte[] ARCHIVE_DATA = new byte[] { 0x01, 0x02, 0x03, 0x04, 0x05 }; - private const string GITHUB_ORG = "target-org"; - private const string GITHUB_REPO = "target-repo"; - private const string GITHUB_PAT = "github pat"; - private const string AWS_BUCKET_NAME = "aws-bucket-name"; - private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; - private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; - private const string AWS_SESSION_TOKEN = "aws-session-token"; - private const string AWS_REGION = "eu-west-1"; - private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; - - private const string BBS_HOST = "our-bbs-server.com"; - private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; - private const string BBS_PROJECT = "bbs-project"; - private const string BBS_REPO = "bbs-repo"; - private const string BBS_REPO_URL = $"{BBS_SERVER_URL}/projects/{BBS_PROJECT}/repos/{BBS_REPO}/browse"; - private const string UNUSED_REPO_URL = "https://not-used"; - private const string SSH_USER = "ssh-user"; - private const string PRIVATE_KEY = "private-key"; - private const string SMB_USER = "smb-user"; - private const string SMB_PASSWORD = "smb-password"; - private const long BBS_EXPORT_ID = 123; - - private const string GITHUB_ORG_ID = "github-org-id"; - private const string MIGRATION_SOURCE_ID = "migration-source-id"; - private const string MIGRATION_ID = "migration-id"; - - public MigrateRepoCommandHandlerTests() - { - _warningsCountLogger = new WarningsCountLogger(_mockOctoLogger.Object); - _handler = new MigrateRepoCommandHandler( - _mockOctoLogger.Object, - _mockGithubApi.Object, - _mockGitlabApi.Object, - _mockEnvironmentVariableProvider.Object, - _mockGitlabArchiveDownloader.Object, - _mockAzureApi.Object, - _mockAwsApi.Object, - _mockFileSystemProvider.Object, - _warningsCountLogger - ); - - // Default setup for file system operations - _mockFileSystemProvider.Setup(m => m.FileExists(It.IsAny())).Returns(true); - _mockFileSystemProvider.Setup(m => m.DirectoryExists(It.IsAny())).Returns(true); - } - - [Fact] - public async Task Happy_Path_Generate_Only() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - }; - await _handler.Handle(args); - - // Assert - _mockGitlabApi.Verify(m => m.StartExport( - BBS_PROJECT, - BBS_REPO - )); - - _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact] - public async Task Happy_Path_Generate_And_Download() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - }; - await _handler.Handle(args); - - // Assert - _mockGitlabArchiveDownloader.Verify(m => m.Download(BBS_EXPORT_ID, It.IsAny())); - _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact] - public async Task Happy_Path_Ingest_Only() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - UNUSED_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Happy_Path_Generate_Archive_Ssh_Download_Azure_Upload_And_Ingest() - { - // Arrange - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - BBS_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Happy_Path_Generate_Archive_Ssh_Download_Aws_Upload_And_Ingest() - { - // Arrange - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AwsBucketName = AWS_BUCKET_NAME, - AwsAccessKey = AWS_ACCESS_KEY_ID, - AwsSecretKey = AWS_SECRET_ACCESS_KEY, - AwsRegion = AWS_REGION, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - BBS_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Happy_Path_Full_Flow_Running_On_Gitlab_Server() - { - // Arrange - const string bbsSharedHome = "bbs-shared-home"; - var archivePath = $"{bbsSharedHome}/data/migration/export/Bitbucket_export_{BBS_EXPORT_ID}.tar"; - - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - GitlabSharedHome = bbsSharedHome, - QueueOnly = true, - }; - - var handler = new MigrateRepoCommandHandler( - _mockOctoLogger.Object, - _mockGithubApi.Object, - _mockGitlabApi.Object, - _mockEnvironmentVariableProvider.Object, - null, // in case of running on Bitbucket server, the downloader will be null - _mockAzureApi.Object, - _mockAwsApi.Object, - _mockFileSystemProvider.Object, - _warningsCountLogger - ); - await handler.Handle(args); - - // Assert - args.ArchivePath.Should().Be(archivePath); - - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - BBS_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Happy_Path_Full_Flow_Gitlab_Credentials_Via_Environment() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.GitlabUsername(It.IsAny())).Returns(BBS_USERNAME); - _mockEnvironmentVariableProvider.Setup(m => m.GitlabPassword(It.IsAny())).Returns(BBS_PASSWORD); - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO).Result).Returns(false); - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - BBS_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Happy_Path_Uploads_To_Github_Storage() - { - // Arrange - var githubOrgDatabaseId = Guid.NewGuid().ToString(); - const string gitArchiveFilePath = "./gitdata_archive"; - const string gitArchiveUrl = "gei://archive/1"; - const string gitArchiveContents = "I am git archive"; - - await using var gitContentStream = new MemoryStream(gitArchiveContents.ToBytes()); - - _mockFileSystemProvider.Setup(m => m.OpenRead(gitArchiveFilePath)).Returns(gitContentStream); - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - _mockGithubApi.Setup(x => x.GetOrganizationDatabaseId(GITHUB_ORG).Result).Returns(githubOrgDatabaseId); - _mockGithubApi - .Setup(x => x.UploadArchiveToGithubStorage( - githubOrgDatabaseId, - It.IsAny(), - It.Is(s => (s as MemoryStream).ToArray().GetString() == gitArchiveContents)).Result) - .Returns(gitArchiveUrl); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - ArchivePath = gitArchiveFilePath, - UseGithubStorage = true, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - BBS_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - gitArchiveUrl, - null - )); - } - - [Fact] - public async Task Happy_Path_Deletes_Downloaded_Archive() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); - } - - [Fact] - public async Task It_Deletes_Downloaded_Archive_Even_If_Upload_Fails() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ThrowsAsync(new InvalidOperationException()); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT - }; - await _handler.Invoking(async x => await x.Handle(args)).Should().ThrowExactlyAsync(); - - // Assert - _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); - } - - [Fact] - public async Task Happy_Path_Does_Not_Throw_If_Fails_To_Delete_Downloaded_Archive() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(ARCHIVE_DATA); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - _mockFileSystemProvider.Setup(x => x.DeleteIfExists(It.IsAny())).Throws(new UnauthorizedAccessException("Access Denied")); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockFileSystemProvider.Verify(x => x.DeleteIfExists(ARCHIVE_PATH)); - } - - [Fact] - public async Task Dont_Generate_Archive_If_Target_Repo_Exists() - { - // Arrange - _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(true); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = GITHUB_PAT, - QueueOnly = true, - }; - - await FluentActions - .Invoking(async () => await _handler.Handle(args)).Should().ThrowExactlyAsync(); - - // Assert - _mockGitlabApi.Verify(x => x.StartExport(It.IsAny(), It.IsAny()), Times.Never); - } - - [Fact] - public async Task Uses_GitHub_Pat_When_Provided_As_Option() - { - // Arrange - var githubPat = "specific github pat"; - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - _mockGithubApi - .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) - .Returns(MIGRATION_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GithubPat = githubPat, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - UNUSED_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - githubPat, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Skip_Migration_If_Target_Repo_Exists() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - _mockGithubApi - .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) - .Throws(new OctoshiftCliException($"A repository called {GITHUB_ORG}/{GITHUB_REPO} already exists")); - - // Act - var args = new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockOctoLogger.Verify(m => m.LogWarning(It.IsAny()), Times.Exactly(1)); - } - - [Fact] - public async Task Throws_Decorated_Error_When_Create_Migration_Source_Fails_With_Permissions_Error() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi - .Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result) - .Throws(new OctoshiftCliException("monalisa does not have the correct permissions to execute `CreateMigrationSource`")); - - // Act - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - })) - .Should() - .ThrowAsync() - .WithMessage($"monalisa does not have the correct permissions to execute `CreateMigrationSource`. Please check that:\n (a) you are a member of the `{GITHUB_ORG}` organization,\n (b) you are an organization owner or you have been granted the migrator role and\n (c) your personal access token has the correct scopes.\nFor more information, see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer."); - } - - [Fact] - public async Task Throws_An_Error_If_Export_Fails() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("FAILED", "The export failed", 0)); - - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - }; - - // Assert - await _handler.Invoking(x => x.Handle(args)).Should().ThrowExactlyAsync(); - } - - [Fact] - public async Task Uses_Archive_Path_If_Provided() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); - - var archiveBytes = Encoding.ASCII.GetBytes("here are some bytes"); - _mockFileSystemProvider.Setup(x => x.ReadAllBytesAsync(ARCHIVE_PATH)).ReturnsAsync(archiveBytes); - - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - _mockGithubApi - .Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null).Result) - .Returns(MIGRATION_ID); - - // Act - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - UNUSED_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - } - - [Fact] - public async Task Invoke_With_Gitlab_Server_Url_Throws_When_Smb_User_Is_Provided_And_Smb_Password_Is_Not_Provided() - { - // Act, Assert - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - SmbUser = SMB_USER - }; - - await _handler.Invoking(x => x.Handle(args)).Should().ThrowExactlyAsync(); - } - - [Fact] - public async Task Invoke_With_Gitlab_Server_Url_Should_Not_Throw_When_Smb_User_Is_Provided_And_Smb_Password_Is_Provided_Via_Environment_Variable() - { - // Arrange - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - - _mockEnvironmentVariableProvider.Setup(m => m.SmbPassword(It.IsAny())).Returns(SMB_PASSWORD); - - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - }; - - // Act, Assert - await _handler.Invoking(x => x.Handle(args)).Should().NotThrowAsync(); - } - - [Fact] - public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Path_Is_Provided() - { - // Arrange - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - - // Act - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - args.ArchivePath.Should().Be(ARCHIVE_PATH); - _mockFileSystemProvider.Verify(m => m.OpenRead(ARCHIVE_PATH)); - } - - [Fact] - public async Task It_Does_Not_Set_The_Archive_Path_When_Archive_Url_Is_Provided() - { - // Act - var args = new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - }; - await _handler.Handle(args); - - // Assert - args.ArchivePath.Should().BeNull(); - _mockFileSystemProvider.Verify(m => m.ReadAllBytesAsync(It.IsAny()), Times.Never); - } - - [Fact] - public async Task Uses_Aws_If_Credentials_Are_Passed() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); - - _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG).Result).Returns(GITHUB_ORG_ID); - _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID).Result).Returns(MIGRATION_SOURCE_ID); - _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); - - // Act - var args = new MigrateRepoCommandArgs - { - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - ArchivePath = ARCHIVE_PATH, - AwsAccessKey = AWS_ACCESS_KEY_ID, - AwsSecretKey = AWS_SECRET_ACCESS_KEY, - AwsBucketName = AWS_BUCKET_NAME, - AwsRegion = AWS_REGION, - QueueOnly = true, - }; - - await _handler.Handle(args); - - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - MIGRATION_SOURCE_ID, - UNUSED_REPO_URL, - GITHUB_ORG_ID, - GITHUB_REPO, - GITHUB_PAT, - ARCHIVE_URL, - null - )); - - _mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())); - } - - [Fact] - public async Task It_Throws_When_Both_Azure_Storage_Connection_String_And_Aws_Bucket_Name_Are_Not_Provided() - { - await _handler.Invoking(async x => await x.Handle( - new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - })) - .Should() - .ThrowAsync() - .WithMessage("*--azure-storage-connection-string*AZURE_STORAGE_CONNECTION_STRING*or*--aws-bucket-name*--aws-access-key*AWS_ACCESS_KEY_ID*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); - } - - [Fact] - public async Task It_Throws_When_Both_Azure_Storage_Connection_String_And_Aws_Bucket_Name_Are_Provided() - { - await _handler.Invoking(async x => await x.Handle( - new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - AwsBucketName = AWS_BUCKET_NAME - })) - .Should() - .ThrowAsync() - .WithMessage("*--azure-storage-connection-string*AZURE_STORAGE_CONNECTION_STRING*and*--aws-bucket-name*--aws-access-key*AWS_ACCESS_KEY_ID*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); - } - - [Fact] - public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_But_No_Aws_Access_Key_Id() - { - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AwsBucketName = AWS_BUCKET_NAME - })) - .Should() - .ThrowAsync() - .WithMessage("*--aws-access-key*AWS_ACCESS_KEY_ID*"); - } - - [Fact] - public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Secret_Access_Key() - { - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AwsBucketName = AWS_BUCKET_NAME, - AwsAccessKey = AWS_ACCESS_KEY_ID - })) - .Should() - .ThrowAsync() - .WithMessage("*--aws-secret-key*AWS_SECRET_ACCESS_KEY*"); - } - - [Fact] - public async Task It_Throws_When_Aws_Bucket_Name_Is_Provided_But_No_Aws_Region() - { - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AwsBucketName = AWS_BUCKET_NAME, - AwsAccessKey = AWS_ACCESS_KEY_ID, - AwsSecretKey = AWS_SECRET_ACCESS_KEY, - AwsSessionToken = AWS_SESSION_TOKEN - })) - .Should() - .ThrowAsync() - .WithMessage("Either --aws-region or AWS_REGION environment variable must be set."); - } - - [Fact] - public async Task Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Username() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO - }; - - // Assert - await _handler.Invoking(x => x.Handle(args)) - .Should() - .ThrowExactlyAsync() - .WithMessage("*BBS_USERNAME*--bbs-username*"); - } + _warningsCountLogger = new WarningsCountLogger(_mockOctoLogger.Object); + _handler = new MigrateRepoCommandHandler( + _mockOctoLogger.Object, + _mockGithubApi.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object, + _mockAzureApi.Object, + _mockAwsApi.Object, + _mockHttpDownloadService.Object, + _mockFileSystemProvider.Object, + _warningsCountLogger + ); + + _mockFileSystemProvider.Setup(m => m.FileExists(It.IsAny())).Returns(true); + _mockFileSystemProvider.Setup(m => m.GetTempFileName()).Returns(ARCHIVE_PATH); + } - [Fact] - public async Task Errors_If_GitlabServer_Url_Provided_But_No_Gitlab_Password() - { - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - GitlabUsername = BBS_USERNAME - }; + [Fact] + public async Task Throws_If_Args_Is_Null() + { + await FluentActions + .Invoking(() => _handler.Handle(null)) + .Should() + .ThrowExactlyAsync(); + } - // Assert - await _handler.Invoking(x => x.Handle(args)) - .Should() - .ThrowExactlyAsync() - .WithMessage("*BBS_PASSWORD*--bbs-password*"); - } + [Fact] + public async Task Throws_If_Target_Repo_Already_Exists() + { + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(true); - [Fact] - public async Task It_Should_Not_Validate_Gitlab_Username_And_Password_When_Kerberos_Is_Set() + var args = new MigrateRepoCommandArgs { - // Arrange - _mockGitlabApi.Setup(x => x.GetExport(It.IsAny())).ReturnsAsync(("COMPLETED", "The export is complete", 100)); + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + }; - // Act - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - Kerberos = true - }; + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"A repository called {GITHUB_ORG}/{GITHUB_REPO} already exists"); + } - // Assert - await _handler.Invoking(x => x.Handle(args)) - .Should() - .NotThrowAsync(); - } + [Fact] + public async Task Generate_Only_Calls_Start_Export_And_Downloads() + { + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)) + .ReturnsAsync(("finished", ARCHIVE_URL)); - [Fact] - public async Task Sets_Target_Repo_Visibility() + var args = new MigrateRepoCommandArgs { - // Arrange - var targetRepoVisibility = "public"; - - // Act - var args = new MigrateRepoCommandArgs - { - ArchiveUrl = ARCHIVE_URL, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - TargetRepoVisibility = targetRepoVisibility, - }; - await _handler.Handle(args); + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; - // Assert - _mockGithubApi.Verify(m => m.StartGitlabMigration( - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - It.IsAny(), - targetRepoVisibility - )); - } + await _handler.Handle(args); - [Fact] - public async Task It_Throws_When_Archive_Path_Does_Not_Exist() - { - const string nonExistentArchivePath = "/path/to/nonexistent/archive.tar"; - _mockFileSystemProvider.Setup(m => m.FileExists(nonExistentArchivePath)).Returns(false); + _mockGitlabApi.Verify(m => m.StartExport(GITLAB_GROUP, GITLAB_PROJECT)); + _mockGitlabApi.Verify(m => m.DownloadExportArchive(GITLAB_GROUP, GITLAB_PROJECT, It.IsAny())); + _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); + } - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - ArchivePath = nonExistentArchivePath, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING - })) - .Should() - .ThrowAsync() - .WithMessage($"*--archive-path*{nonExistentArchivePath}*"); - } + [Fact] + public async Task Ingest_Only_Starts_Gitlab_Migration_With_Unused_Source_Url() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } - [Fact] - public async Task It_Throws_When_Gitlab_Shared_Home_Does_Not_Exist_When_Running_On_Bitbucket_Instance() - { - const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); + [Fact] + public async Task Passes_Gitlab_Project_Url_When_All_Gitlab_Args_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, GITLAB_PROJECT_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)).ReturnsAsync(("finished", ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + GITLAB_PROJECT_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } - await _handler.Invoking(async x => await x.Handle(new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabSharedHome = nonExistentGitlabSharedHome - })) - .Should() - .ThrowAsync() - .WithMessage($"*--bbs-shared-home*{nonExistentGitlabSharedHome}*"); - } + [Fact] + public async Task Uploads_To_Aws_When_Aws_Bucket_Name_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.AwsAccessKeyId(It.IsAny())).Returns(AWS_ACCESS_KEY_ID); + _mockEnvironmentVariableProvider.Setup(m => m.AwsSecretAccessKey(It.IsAny())).Returns(AWS_SECRET_ACCESS_KEY); + _mockEnvironmentVariableProvider.Setup(m => m.AwsRegion(It.IsAny())).Returns(AWS_REGION); + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsRegion = AWS_REGION, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())); + } - [Fact] - public async Task It_Does_Not_Validate_Gitlab_Shared_Home_When_Using_Ssh() - { - const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); + [Fact] + public async Task Deletes_Archive_By_Default() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); + } - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabSharedHome = nonExistentGitlabSharedHome, - SshUser = SSH_USER, - SshPrivateKey = PRIVATE_KEY - }; + [Fact] + public async Task Keeps_Archive_When_KeepArchive_Set() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + KeepArchive = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(It.IsAny()), Times.Never); + } - await _handler.Invoking(x => x.Handle(args)) - .Should() - .NotThrowAsync(); - } + [Fact] + public async Task Throws_When_Gitlab_Pat_Not_Provided_For_Generate() + { + _mockEnvironmentVariableProvider.Setup(m => m.GitlabPat(It.IsAny())).Returns((string)null); - [Fact] - public async Task It_Does_Not_Validate_Gitlab_Shared_Home_When_Using_Smb() + var args = new MigrateRepoCommandArgs { - const string nonExistentGitlabSharedHome = "/nonexistent/shared/home"; - _mockGitlabApi.Setup(x => x.StartExport(BBS_PROJECT, BBS_REPO)).ReturnsAsync(BBS_EXPORT_ID); - _mockGitlabApi.Setup(x => x.GetExport(BBS_EXPORT_ID)).ReturnsAsync(("COMPLETED", "The export is complete", 100)); - _mockGitlabArchiveDownloader.Setup(x => x.Download(BBS_EXPORT_ID, It.IsAny())).ReturnsAsync(ARCHIVE_PATH); - _mockFileSystemProvider.Setup(m => m.DirectoryExists(nonExistentGitlabSharedHome)).Returns(false); - _mockEnvironmentVariableProvider.Setup(m => m.SmbPassword(It.IsAny())).Returns(SMB_PASSWORD); + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - GitlabProject = BBS_PROJECT, - GitlabRepo = BBS_REPO, - GitlabSharedHome = nonExistentGitlabSharedHome, - SmbUser = SMB_USER - }; + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*GitLab PAT*GITLAB_PAT*--gitlab-pat*"); + } - await _handler.Invoking(x => x.Handle(args)) - .Should() - .NotThrowAsync(); - } + [Fact] + public async Task Throws_When_Archive_Path_Does_Not_Exist() + { + _mockFileSystemProvider.Setup(m => m.FileExists(ARCHIVE_PATH)).Returns(false); - [Fact] - public async Task It_Logs_Archive_Path_Before_Upload() + var args = new MigrateRepoCommandArgs { - _mockFileSystemProvider.Setup(m => m.FileExists(ARCHIVE_PATH)).Returns(true); - _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); - - var args = new MigrateRepoCommandArgs - { - ArchivePath = ARCHIVE_PATH, - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, - GithubOrg = GITHUB_ORG, - GithubRepo = GITHUB_REPO, - QueueOnly = true, - }; + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + }; - await _handler.Handle(args); - _mockOctoLogger.Verify(m => m.LogInformation($"Archive path: {ARCHIVE_PATH}"), Times.Once); - } + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*archive*--archive-path*"); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs index 98b2550a8..7b433b1dc 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs @@ -1,10 +1,11 @@ using System; +using System.Net.Http; using FluentAssertions; using Moq; -using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; -using OctoshiftCLI.GitlabToGithub.Factories; using OctoshiftCLI.Contracts; using OctoshiftCLI.Factories; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.GitlabToGithub.Factories; using OctoshiftCLI.Services; using Xunit; @@ -12,21 +13,11 @@ namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; public class MigrateRepoCommandTests { - private const string ARCHIVE_DOWNLOAD_HOST = "archive-download-host"; - private const string SSH_USER = "ssh-user"; - private const string SSH_PRIVATE_KEY = "ssh-private-key"; - private const int SSH_PORT = 1234; - private const string BBS_SHARED_HOME = "shared-home"; - private const string BBS_HOST = "bbs-host"; - private const string BBS_SERVER_URL = $"https://{BBS_HOST}"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; private const string GITHUB_ORG = "github-org"; private const string GITHUB_PAT = "github-pat"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; - private const string SMB_USER = "smb-user"; - private const string SMB_PASSWORD = "smb-password"; - private const string SMB_DOMAIN = "smb-domain"; private readonly Mock _mockServiceProvider = new(); private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); @@ -34,7 +25,6 @@ public class MigrateRepoCommandTests private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); private readonly Mock _mockGithubApiFactory = new(); private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); - private readonly Mock _mockGitlabArchiveDownloaderFactory = TestHelpers.CreateMock(); private readonly Mock _mockAzureApiFactory = new(); private readonly Mock _warningsCountLogger = TestHelpers.CreateMock(); @@ -47,8 +37,13 @@ public MigrateRepoCommandTests() _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); _mockServiceProvider.Setup(m => m.GetService(typeof(ITargetGithubApiFactory))).Returns(_mockGithubApiFactory.Object); _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); - _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabArchiveDownloaderFactory))).Returns(_mockGitlabArchiveDownloaderFactory.Object); _mockServiceProvider.Setup(m => m.GetService(typeof(IAzureApiFactory))).Returns(_mockAzureApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(HttpDownloadServiceFactory))) + .Returns(new HttpDownloadServiceFactory( + _mockOctoLogger.Object, + new Mock().Object, + _mockFileSystemProvider.Object, + new Mock().Object)); _mockServiceProvider.Setup(m => m.GetService(typeof(WarningsCountLogger))).Returns(_warningsCountLogger.Object); } @@ -58,13 +53,12 @@ public void Should_Have_Options() var command = new MigrateRepoCommand(); command.Should().NotBeNull(); command.Name.Should().Be("migrate-repo"); - command.Options.Count.Should().Be(33); + command.Options.Count.Should().Be(23); - TestHelpers.VerifyCommandOption(command.Options, "bbs-server-url", true); - TestHelpers.VerifyCommandOption(command.Options, "bbs-project", true); - TestHelpers.VerifyCommandOption(command.Options, "bbs-repo", true); - TestHelpers.VerifyCommandOption(command.Options, "bbs-username", false); - TestHelpers.VerifyCommandOption(command.Options, "bbs-password", false); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-group", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-project", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-pat", false); TestHelpers.VerifyCommandOption(command.Options, "archive-url", false); TestHelpers.VerifyCommandOption(command.Options, "archive-path", false); TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false); @@ -76,16 +70,8 @@ public void Should_Have_Options() TestHelpers.VerifyCommandOption(command.Options, "github-org", false); TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); - TestHelpers.VerifyCommandOption(command.Options, "archive-download-host", false); - TestHelpers.VerifyCommandOption(command.Options, "ssh-user", false); - TestHelpers.VerifyCommandOption(command.Options, "ssh-private-key", false); - TestHelpers.VerifyCommandOption(command.Options, "ssh-port", false); - TestHelpers.VerifyCommandOption(command.Options, "smb-user", false); - TestHelpers.VerifyCommandOption(command.Options, "smb-password", false); - TestHelpers.VerifyCommandOption(command.Options, "smb-domain", false); TestHelpers.VerifyCommandOption(command.Options, "queue-only", false); TestHelpers.VerifyCommandOption(command.Options, "target-repo-visibility", false); - TestHelpers.VerifyCommandOption(command.Options, "kerberos", false, true); TestHelpers.VerifyCommandOption(command.Options, "verbose", false); TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false); TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false); @@ -94,135 +80,37 @@ public void Should_Have_Options() TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true); } - [Fact] - public void BuildHandler_Creates_Gitlab_Ssh_Archive_Downloader_Based_On_Server_Url_When_Ssh_User_Is_Provided() - { - // Arrange - var args = new MigrateRepoCommandArgs - { - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - GitlabSharedHome = BBS_SHARED_HOME, - GitlabServerUrl = BBS_SERVER_URL - }; - - // Act - var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - - // Assert - handler.Should().NotBeNull(); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(BBS_HOST, SSH_USER, SSH_PRIVATE_KEY, SSH_PORT, BBS_SHARED_HOME)); - } - - [Fact] - public void BuildHandler_Creates_Gitlab_Ssh_Archive_Downloader_When_Ssh_User_And_Archive_Download_Host_Is_Provided() - { - // Arrange - var args = new MigrateRepoCommandArgs - { - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SshUser = SSH_USER, - SshPrivateKey = SSH_PRIVATE_KEY, - SshPort = SSH_PORT, - GitlabSharedHome = BBS_SHARED_HOME, - GitlabServerUrl = BBS_SERVER_URL - }; - - // Act - var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - - // Assert - handler.Should().NotBeNull(); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(ARCHIVE_DOWNLOAD_HOST, SSH_USER, SSH_PRIVATE_KEY, SSH_PORT, BBS_SHARED_HOME)); - } - - [Fact] - public void BuildHandler_Creates_Gitlab_Smb_Archive_Downloader_Based_On_Server_Url_When_Smb_User_Is_Provided() - { - // Arrange - var args = new MigrateRepoCommandArgs - { - SmbUser = SMB_USER, - SmbPassword = SMB_PASSWORD, - SmbDomain = SMB_DOMAIN, - GitlabSharedHome = BBS_SHARED_HOME, - GitlabServerUrl = BBS_SERVER_URL - }; - - // Act - var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - - // Assert - handler.Should().NotBeNull(); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(BBS_HOST, SMB_USER, SMB_PASSWORD, SMB_DOMAIN, BBS_SHARED_HOME)); - } - - [Fact] - public void BuildHandler_Creates_Gitlab_Smb_Archive_Downloader_When_Smb_User_And_Archive_Download_Host_Is_Provided() - { - // Arrange - var args = new MigrateRepoCommandArgs - { - ArchiveDownloadHost = ARCHIVE_DOWNLOAD_HOST, - SmbUser = SMB_USER, - SmbPassword = SMB_PASSWORD, - SmbDomain = SMB_DOMAIN, - GitlabSharedHome = BBS_SHARED_HOME, - GitlabServerUrl = BBS_SERVER_URL - }; - - // Act - var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - - // Assert - handler.Should().NotBeNull(); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(ARCHIVE_DOWNLOAD_HOST, SMB_USER, SMB_PASSWORD, SMB_DOMAIN, BBS_SHARED_HOME)); - } - [Fact] public void BuildHandler_Creates_The_Handler() { - // Arrange var args = new MigrateRepoCommandArgs(); - // Act var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - // Assert handler.Should().NotBeNull(); - - _mockGithubApiFactory.Verify(m => m.Create(It.IsAny(), null, It.IsAny()), Times.Never); - _mockGitlabApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSshDownloader(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); - _mockGitlabArchiveDownloaderFactory.Verify(m => m.CreateSmbDownloader(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGithubApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGitlabApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); _mockAzureApiFactory.Verify(m => m.Create(It.IsAny()), Times.Never); - _mockAzureApiFactory.Verify(m => m.CreateClientNoSsl(It.IsAny()), Times.Never); } [Fact] public void BuildHandler_Creates_GitHub_Api_When_Github_Org_Is_Provided() { - // Arrange var args = new MigrateRepoCommandArgs { GithubOrg = GITHUB_ORG, GithubPat = GITHUB_PAT }; - // Act var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - // Assert handler.Should().NotBeNull(); - _mockGithubApiFactory.Verify(m => m.Create(null, null, GITHUB_PAT)); } [Fact] public void BuildHandler_Uses_Target_Api_Url_When_Provided() { - // Arrange var targetApiUrl = "https://api.github.com"; var args = new MigrateRepoCommandArgs { @@ -231,127 +119,54 @@ public void BuildHandler_Uses_Target_Api_Url_When_Provided() TargetApiUrl = targetApiUrl }; - // Act var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - // Assert handler.Should().NotBeNull(); - _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, GITHUB_PAT)); } [Fact] public void BuildHandler_Creates_Gitlab_Api_When_Gitlab_Server_Url_Is_Provided() { - // Arrange var args = new MigrateRepoCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT }; - // Act var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - // Assert handler.Should().NotBeNull(); - - _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, false)); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); } [Fact] - public void BuildHandler_Creates_Azure_Api_Factory_When_Azure_Storage_Connection_String_Is_Provided_Via_Args() + public void BuildHandler_Forwards_NoSslVerify_To_Gitlab_Api_Factory() { - // Arrange var args = new MigrateRepoCommandArgs { - AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = true }; - // Act var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - // Assert handler.Should().NotBeNull(); - - _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); - } - - [Fact] - public void BuildHandler_Creates_Azure_Api_Factory_When_Azure_Storage_Connection_String_Is_Provided_Via_Environment_Variables() - { - // Arrange - _mockEnvironmentVariableProvider.Setup(m => m.AzureStorageConnectionString(false)).Returns(AZURE_STORAGE_CONNECTION_STRING); - - var args = new MigrateRepoCommandArgs(); - - // Act - var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - - // Assert - handler.Should().NotBeNull(); - - _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); - } - - [Fact] - public void It_Gets_A_Kerberos_HttpClient_When_Kerberos_Is_True() - { - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - Kerberos = true - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, false)); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, true)); } [Fact] - public void It_Gets_A_Kerberos_With_No_Ssl_Verify_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_True() + public void BuildHandler_Creates_Azure_Api_When_Connection_String_Is_Provided_Via_Args() { var args = new MigrateRepoCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - Kerberos = true, - NoSslVerify = true - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.CreateKerberos(BBS_SERVER_URL, true)); - } - - [Fact] - public void It_Gets_A_Default_HttpClient_When_Kerberos_And_No_Ssl_Verify_Are_Not_Set() - { - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD - }; - - _command.BuildHandler(args, _mockServiceProvider.Object); - - _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, false)); - } - - [Fact] - public void It_Gets_A_No_Ssl_Verify_HttpClient_When_No_Ssl_Verify_Is_True() - { - var args = new MigrateRepoCommandArgs - { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, - NoSslVerify = true + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING }; - _command.BuildHandler(args, _mockServiceProvider.Object); + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); - _mockGitlabApiFactory.Verify(m => m.Create(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, true)); + handler.Should().NotBeNull(); + _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); } } diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs index aa5626b52..ac4e29925 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -52,7 +52,7 @@ public MigrateRepoCommand() : base( public Option GitlabGroup { get; } = new( name: "--gitlab-group", - description: "The GitLab group to migrate.") + description: "The GitLab group (full namespace path) that contains the project to migrate. For nested subgroups, use the full path, e.g. parent-group/subgroup.") { IsRequired = true }; @@ -75,7 +75,7 @@ public MigrateRepoCommand() : base( public Option ArchivePath { get; } = new( name: "--archive-path", - description: "Path to the GitLab migration archive on disk."); + description: "Path to the GitLab migration archive on disk. When --gitlab-server-url is provided, the generated archive will be written to this path (overwriting any existing file)."); public Option AzureStorageConnectionString { get; } = new( name: "--azure-storage-connection-string", diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs index 220ea9584..603dfb36e 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -86,7 +86,7 @@ private void ValidateNoGenerateOptions() } } - public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchivePath.HasValue() && !ArchiveUrl.HasValue(); + public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue(); public bool ShouldUploadArchive() => ArchiveUrl.IsNullOrWhiteSpace() && GithubOrg.HasValue(); diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index fd99a86c0..b8dd659ad 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -280,7 +280,7 @@ private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => a private string GetGitlabProjectUrl(MigrateRepoCommandArgs args) { return args.GitlabServerUrl.HasValue() && args.GitlabGroup.HasValue() && args.GitlabProject.HasValue() - ? $"{args.GitlabServerUrl.TrimEnd('/')}/projects/{args.GitlabGroup}/repos/{args.GitlabProject}/browse" + ? $"{args.GitlabServerUrl.TrimEnd('/')}/{args.GitlabGroup}/{args.GitlabProject}" : "https://not-used"; } @@ -294,8 +294,8 @@ private void ValidateOptions(MigrateRepoCommandArgs args) } } - // Validate --archive-path if provided - if (args.ArchivePath.HasValue() && !_fileSystemProvider.FileExists(args.ArchivePath)) + // Validate --archive-path if provided as an input (i.e. not generating a new archive) + if (!args.ShouldGenerateArchive() && args.ArchivePath.HasValue() && !_fileSystemProvider.FileExists(args.ArchivePath)) { throw new OctoshiftCliException($"The archive file provided with --archive-path does not exist or is not accessible: {args.ArchivePath}"); } From fd6b9bb14d51389231cc09ead74bf28f78d0034e Mon Sep 17 00:00:00 2001 From: Briana J Date: Wed, 20 May 2026 19:23:15 +0000 Subject: [PATCH 70/71] fix inventory report command --- src/Octoshift/Models/GitlabProject.cs | 2 +- src/Octoshift/Services/GithubApi.cs | 2 +- src/Octoshift/Services/GitlabClient.cs | 30 ++--- .../Octoshift/Services/GithubApiTests.cs | 109 ++++++++++++++++++ .../InventoryReportCommandHandlerTests.cs | 92 ++++++++------- .../InventoryReportCommandTests.cs | 44 ++++--- .../ProjectsCsvGeneratorServiceTests.cs | 4 +- src/gl2gh/Services/GitlabInspectorService.cs | 4 +- 8 files changed, 207 insertions(+), 80 deletions(-) diff --git a/src/Octoshift/Models/GitlabProject.cs b/src/Octoshift/Models/GitlabProject.cs index 10b337d65..bee22dd75 100644 --- a/src/Octoshift/Models/GitlabProject.cs +++ b/src/Octoshift/Models/GitlabProject.cs @@ -5,5 +5,5 @@ public record GitlabProject public string Id { get; init; } public string Name { get; init; } public string Path { get; init; } - public string Archived { get; init; } + public bool Archived { get; init; } } diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index bf4b2ff53..7e42442cb 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -568,7 +568,7 @@ public virtual async Task StartGitlabMigration(string migrationSourceId, "not-used", // source access token targetToken, archiveUrl, - "https://not-used", // metadata archive URL + null, // GitLab archive contains both git and metadata — GitHub falls back to gitArchiveUrl false, // skip releases targetRepoVisibility, false // lock source diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs index f7c9176dc..353021873 100644 --- a/src/Octoshift/Services/GitlabClient.cs +++ b/src/Octoshift/Services/GitlabClient.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Collections.Generic; +using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -55,20 +56,22 @@ public virtual async Task GetAsync(string url) public virtual async IAsyncEnumerable GetAllAsync(string url) { - var hasNextPage = true; - var nextPageStart = 0; - while (hasNextPage) + var nextPage = 1; + while (nextPage > 0) { - var response = await GetWithPagination(url, nextPageStart); - var jResponse = JObject.Parse(response); + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, AddPageParam(url, nextPage))); + var content = await response.Content.ReadAsStringAsync(); + var jArray = JArray.Parse(content); - foreach (var jToken in jResponse["values"]!) + foreach (var jToken in jArray) { yield return jToken; } - hasNextPage = !jResponse["isLastPage"]?.ToObject() ?? false; - nextPageStart = jResponse["nextPageStart"]?.ToObject() ?? 0; + nextPage = response.Headers.TryGetValues("X-Next-Page", out var values) + && int.TryParse(values.FirstOrDefault(), out var parsed) + ? parsed + : 0; } } @@ -89,8 +92,6 @@ public virtual async Task DeleteAsync(string url) return await response.Content.ReadAsStringAsync(); } - private async Task GetWithPagination(string url, int start = 0, int limit = DEFAULT_PAGE_SIZE) => await GetAsync(AddPaginationParams(url, start, limit)); - private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) { _log.LogVerbose($"HTTP {httpMethod}: {url}"); @@ -118,14 +119,17 @@ private async Task SendAsync(HttpMethod httpMethod, string return response; } - private string AddPaginationParams(string url, int start, int limit) + private static string AddPageParam(string url, int page) { var uri = new Uri(url); var path = uri.GetLeftPart(UriPartial.Path); var queryParams = HttpUtility.ParseQueryString(uri.Query); - queryParams["start"] = start.ToString(); - queryParams["limit"] = limit.ToString(); + queryParams["page"] = page.ToString(); + if (string.IsNullOrEmpty(queryParams["per_page"])) + { + queryParams["per_page"] = DEFAULT_PAGE_SIZE.ToString(); + } return $"{path}?{queryParams}"; } diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs index 733621b49..2ca0c36ba 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs @@ -1202,6 +1202,115 @@ mutation startRepositoryMigration( expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); } + [Fact] + public async Task StartGitlabMigration_Sends_Null_Metadata_Url() + { + // Arrange + const string migrationSourceId = "MIGRATION_SOURCE_ID"; + const string sourceRepoUrl = "https://gitlab.com/my-group/my-project"; + const string orgId = "ORG_ID"; + const string url = "https://api.github.com/graphql"; + const string gitArchiveUrl = "GIT_ARCHIVE_URL"; + const string targetToken = "TARGET_TOKEN"; + + const string unusedSourceToken = "not-used"; + + const string query = @" + mutation startRepositoryMigration( + $sourceId: ID!, + $ownerId: ID!, + $sourceRepositoryUrl: URI!, + $repositoryName: String!, + $continueOnError: Boolean!, + $gitArchiveUrl: String, + $metadataArchiveUrl: String, + $accessToken: String!, + $githubPat: String, + $skipReleases: Boolean, + $targetRepoVisibility: String, + $lockSource: Boolean)"; + const string gql = @" + startRepositoryMigration( + input: { + sourceId: $sourceId, + ownerId: $ownerId, + sourceRepositoryUrl: $sourceRepositoryUrl, + repositoryName: $repositoryName, + continueOnError: $continueOnError, + gitArchiveUrl: $gitArchiveUrl, + metadataArchiveUrl: $metadataArchiveUrl, + accessToken: $accessToken, + githubPat: $githubPat, + skipReleases: $skipReleases, + targetRepoVisibility: $targetRepoVisibility, + lockSource: $lockSource + } + ) { + repositoryMigration { + id, + databaseId, + migrationSource { + id, + name, + type + }, + sourceUrl, + state, + failureReason + } + }"; + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + sourceId = migrationSourceId, + ownerId = orgId, + sourceRepositoryUrl = sourceRepoUrl, + repositoryName = GITHUB_REPO, + continueOnError = true, + gitArchiveUrl, + metadataArchiveUrl = (string)null, + accessToken = unusedSourceToken, + githubPat = targetToken, + skipReleases = false, + targetRepoVisibility = (string)null, + lockSource = false + }, + operationName = "startRepositoryMigration" + }; + const string actualRepositoryMigrationId = "RM_kgC4NjFhNmE2NGU2ZWE1YTQwMDA5ODliZjhi"; + var response = JObject.Parse($@" + {{ + ""data"": {{ + ""startRepositoryMigration"": {{ + ""repositoryMigration"": {{ + ""id"": ""{actualRepositoryMigrationId}"", + ""databaseId"": ""3ba25b34-b23d-43fb-a819-f44414be8dc0"", + ""migrationSource"": {{ + ""id"": ""MS_kgC4NjFhNmE2NDViNWZmOTEwMDA5MTZiMGQw"", + ""name"": ""GitLab Source"", + ""type"": ""GITLAB"" + }}, + ""sourceUrl"": ""{sourceRepoUrl}"", + ""state"": ""QUEUED"", + ""failureReason"": """" + }} + }} + }} + }}"); + + _githubClientMock + .Setup(m => m.PostGraphQLAsync(url, It.Is(x => x.ToJson() == payload.ToJson()), null)) + .ReturnsAsync(response); + + // Act + var expectedRepositoryMigrationId = await _githubApi.StartGitlabMigration(migrationSourceId, sourceRepoUrl, orgId, GITHUB_REPO, targetToken, gitArchiveUrl); + + // Assert + expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); + } + [Fact] public async Task StartMigration_Does_Not_Throw_When_Errors_Is_Empty() { diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs index fa398cc1e..ab29c15d5 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs @@ -10,19 +10,18 @@ namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport; public class InventoryReportCommandHandlerTests { - private const string BBS_SERVER_URL = "http://bbs-server-url"; - private const string BBS_PROJECT_KEY = "FP"; - private const string BBS_PROJECT = "foo-project"; - private const string BBS_USERNAME = "bbs-username"; - private const string BBS_PASSWORD = "bbs-password"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "foo-group"; + private const string GITLAB_PAT = "gitlab-pat"; private const bool NO_SSL_VERIFY = true; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGenerator = TestHelpers.CreateMock(); private readonly Mock _mockProjectsCsvGenerator = TestHelpers.CreateMock(); - private readonly Mock _mockReposCsvGenerator = TestHelpers.CreateMock(); + private string _groupsCsvOutput = ""; private string _projectsCsvOutput = ""; - private string _reposCsvOutput = ""; private readonly InventoryReportCommandHandler _handler; @@ -32,19 +31,19 @@ public InventoryReportCommandHandlerTests() TestHelpers.CreateMock().Object, _mockGitlabApi.Object, _mockGitlabInspectorService.Object, - _mockProjectsCsvGenerator.Object, - _mockReposCsvGenerator.Object) + _mockGroupsCsvGenerator.Object, + _mockProjectsCsvGenerator.Object) { WriteToFile = (path, contents) => { - if (path == "projects.csv") + if (path == "groups.csv") { - _projectsCsvOutput = contents; + _groupsCsvOutput = contents; } - if (path == "repos.csv") + if (path == "projects.csv") { - _reposCsvOutput = contents; + _projectsCsvOutput = contents; } return Task.CompletedTask; @@ -55,78 +54,77 @@ public InventoryReportCommandHandlerTests() [Fact] public async Task Happy_Path() { - var expectedProjectsCsv = "csv stuff"; - var expectedReposCsv = "repo csv stuff"; + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; - _mockGitlabApi.Setup(m => m.GetProjects()).ReturnsAsync(new[] { (Id: 1, Key: BBS_PROJECT_KEY, Name: BBS_PROJECT) }); - _mockGitlabInspectorService.Setup(m => m.GetRepoCount()).ReturnsAsync(1); + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); - _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedProjectsCsv); - _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedReposCsv); + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedProjectsCsv); - // var args = new InventoryReportCommandArgs(); var args = new InventoryReportCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, NoSslVerify = NO_SSL_VERIFY }; await _handler.Handle(args); + _groupsCsvOutput.Should().Be(expectedGroupsCsv); _projectsCsvOutput.Should().Be(expectedProjectsCsv); - _reposCsvOutput.Should().Be(expectedReposCsv); } [Fact] - public async Task Scoped_To_Single_Project() + public async Task Scoped_To_Single_Group() { - var expectedProjectsCsv = "csv stuff"; - var expectedReposCsv = "repo csv stuff"; + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; - _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_PROJECT, false)).ReturnsAsync(expectedProjectsCsv); - _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, BBS_PROJECT, false)).ReturnsAsync(expectedReposCsv); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(GITLAB_GROUP)).ReturnsAsync(1); + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedProjectsCsv); var args = new InventoryReportCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GitlabProject = BBS_PROJECT, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabPat = GITLAB_PAT, NoSslVerify = NO_SSL_VERIFY }; await _handler.Handle(args); + _groupsCsvOutput.Should().Be(expectedGroupsCsv); _projectsCsvOutput.Should().Be(expectedProjectsCsv); - _reposCsvOutput.Should().Be(expectedReposCsv); + + _mockGitlabApi.Verify(m => m.GetGroups(), Times.Never); } [Fact] public async Task It_Generates_Minimal_Csvs_When_Requested() { - // Arrange - var expectedProjectsCsv = "csv stuff"; - var expectedReposCsv = "repo csv stuff"; + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); - _mockProjectsCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedProjectsCsv); - _mockReposCsvGenerator.Setup(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedReposCsv); + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedProjectsCsv); - // Act var args = new InventoryReportCommandArgs { - GitlabServerUrl = BBS_SERVER_URL, - GitlabUsername = BBS_USERNAME, - GitlabPassword = BBS_PASSWORD, + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, NoSslVerify = NO_SSL_VERIFY, Minimal = true }; await _handler.Handle(args); - // Assert + _groupsCsvOutput.Should().Be(expectedGroupsCsv); _projectsCsvOutput.Should().Be(expectedProjectsCsv); - _reposCsvOutput.Should().Be(expectedReposCsv); - _mockProjectsCsvGenerator.Verify(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, true)); - _mockReposCsvGenerator.Verify(m => m.Generate(BBS_SERVER_URL, BBS_USERNAME, BBS_PASSWORD, NO_SSL_VERIFY, null, true)); + _mockGroupsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); + _mockProjectsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs index 2e94f7be5..12e738abf 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs @@ -1,3 +1,4 @@ +using FluentAssertions; using Microsoft.Extensions.DependencyInjection; using Moq; using OctoshiftCLI.GitlabToGithub; @@ -10,11 +11,11 @@ namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport { public class InventoryReportCommandTests { - private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGeneratorService = TestHelpers.CreateMock(); private readonly Mock _mockProjectsCsvGeneratorService = TestHelpers.CreateMock(); - private readonly Mock _mockReposCsvGeneratorService = TestHelpers.CreateMock(); private readonly ServiceProvider _serviceProvider; private readonly InventoryReportCommand _command = []; @@ -24,10 +25,10 @@ public InventoryReportCommandTests() var serviceCollection = new ServiceCollection(); serviceCollection .AddSingleton(_mockOctoLogger.Object) - .AddSingleton(_mockGitlabApi.Object) + .AddSingleton(_mockGitlabApiFactory.Object) .AddSingleton(_mockGitlabInspectorServiceFactory.Object) - .AddSingleton(_mockProjectsCsvGeneratorService.Object) - .AddSingleton(_mockReposCsvGeneratorService.Object); + .AddSingleton(_mockGroupsCsvGeneratorService.Object) + .AddSingleton(_mockProjectsCsvGeneratorService.Object); _serviceProvider = serviceCollection.BuildServiceProvider(); } @@ -35,17 +36,32 @@ public InventoryReportCommandTests() [Fact] public void Should_Have_Options() { - Assert.NotNull(_command); - Assert.Equal("inventory-report", _command.Name); - Assert.Equal(7, _command.Options.Count); - - TestHelpers.VerifyCommandOption(_command.Options, "bbs-server-url", true); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-project", false); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-username", false); - TestHelpers.VerifyCommandOption(_command.Options, "bbs-password", false); + _command.Should().NotBeNull(); + _command.Name.Should().Be("inventory-report"); + _command.Options.Count.Should().Be(6); + + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); TestHelpers.VerifyCommandOption(_command.Options, "minimal", false); TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); - TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = "https://gitlab.contoso.com", + GitlabPat = "gitlab-pat" + }; + + var handler = _command.BuildHandler(args, _serviceProvider); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify)); + _mockGitlabInspectorServiceFactory.Verify(m => m.Create(It.IsAny())); } } } diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs index 9f799984e..232d7ef79 100644 --- a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs @@ -58,7 +58,7 @@ public async Task Generate_Returns_Csv_For_Single_Group() var expected = $"{FULL_CSV_HEADER}{Environment.NewLine}" + - $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"\",{mrCount}{Environment.NewLine}"; + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\",{mrCount}{Environment.NewLine}"; result.Should().Be(expected); } @@ -81,7 +81,7 @@ public async Task Generate_Returns_Minimal_Csv_When_Requested() var expected = $"{MINIMAL_CSV_HEADER}{Environment.NewLine}" + - $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"\"{Environment.NewLine}"; + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\"{Environment.NewLine}"; result.Should().Be(expected); _mockGitlabInspectorService.Verify(m => m.GetProjectMergeRequestCount(It.IsAny(), It.IsAny()), Times.Never); diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs index a4f340c1e..b7100510f 100644 --- a/src/gl2gh/Services/GitlabInspectorService.cs +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -48,7 +48,7 @@ public virtual async Task> GetProjects(string groupPa if (!_repos.TryGetValue(groupPath, out var repos)) { repos = (await _gitlabApi.GetProjects(groupPath)) - .Select(repo => new GitlabProject() { Name = repo.Name, Path = repo.Path }) + .Select(repo => new GitlabProject() { Name = repo.Name, Path = repo.Path, Archived = repo.Archived }) .ToList(); _repos.Add(groupPath, repos); } @@ -75,7 +75,7 @@ public virtual async Task GetProjectCount(string groupPath) public virtual async Task GetMergeRequestCount(string groupPath) { var repos = await GetProjects(groupPath); - return await repos.Sum(async repo => await GetProjectMergeRequestCount(groupPath, repo.Name)); + return await repos.Sum(async repo => await GetProjectMergeRequestCount(groupPath, repo.Path)); } public virtual async Task GetProjectMergeRequestCount(string groupPath, string repo) From 2eaaf5e420132f3467210fef78d1aa6add5d599c Mon Sep 17 00:00:00 2001 From: Briana J Date: Wed, 20 May 2026 21:22:14 +0000 Subject: [PATCH 71/71] log version and if CE or not --- src/Octoshift/Services/GitlabApi.cs | 23 ++++- src/Octoshift/Services/GitlabClient.cs | 20 +++++ .../Octoshift/Services/GitlabApiTests.cs | 84 +++++++++++++++++++ .../GenerateScriptCommandHandler.cs | 2 + .../InventoryReportCommandHandler.cs | 2 + .../MigrateRepo/MigrateRepoCommandHandler.cs | 2 + 6 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs index 9edc404c2..bc31a6ec7 100644 --- a/src/Octoshift/Services/GitlabApi.cs +++ b/src/Octoshift/Services/GitlabApi.cs @@ -20,13 +20,24 @@ public GitlabApi(GitlabClient client, string gitlabServerUrl, OctoLogger log) _log = log; } - public virtual async Task GetServerVersion() + public virtual async Task<(string Version, bool Enterprise)> GetServerVersion() { var url = $"{_gitlabBaseUrl}/api/v4/version"; var content = await _client.GetAsync(url); + var data = JObject.Parse(content); - return (string)JObject.Parse(content)["version"]; + return ((string)data["version"], (bool?)data["enterprise"] ?? false); + } + + public virtual async Task LogServerVersion() + { + var (version, enterprise) = await GetServerVersion(); + if (!string.IsNullOrWhiteSpace(version)) + { + var edition = enterprise ? "Enterprise" : "Community"; + _log?.LogInformation($"GitLab version: {version} ({edition} Edition)"); + } } public virtual async Task StartExport(string groupPath, string projectPath) @@ -112,7 +123,13 @@ public virtual async Task GetIsProjectArchived(string groupPath, string pr var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; - var commitsResponse = await _client.GetAsync(url); + // Empty projects (no Git repo yet) return 404 here; treat as no commits. + var commitsResponse = await _client.GetOrNullForNotFoundAsync(url); + if (commitsResponse is null) + { + return null; + } + var commitsData = JArray.Parse(commitsResponse); var lastCommittedDate = (string)commitsData.First?["committed_date"]; diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs index 353021873..f2a0462e7 100644 --- a/src/Octoshift/Services/GitlabClient.cs +++ b/src/Octoshift/Services/GitlabClient.cs @@ -2,6 +2,7 @@ using System.IO; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; @@ -54,6 +55,25 @@ public virtual async Task GetAsync(string url) return await response.Content.ReadAsStringAsync(); } + /// + /// Like , but returns null when the server responds with 404 Not Found + /// instead of throwing. Other HTTP errors are still retried and bubble up as exceptions. + /// + public virtual async Task GetOrNullForNotFoundAsync(string url) + { + try + { + using var response = await _retryPolicy.HttpRetry( + async () => await SendAsync(HttpMethod.Get, url), + ex => ex.StatusCode != HttpStatusCode.NotFound); + return await response.Content.ReadAsStringAsync(); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + public virtual async IAsyncEnumerable GetAllAsync(string url) { var nextPage = 1; diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs new file mode 100644 index 000000000..badc153b8 --- /dev/null +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs @@ -0,0 +1,84 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.Octoshift.Services; + +public class GitlabApiTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabClient = TestHelpers.CreateMock(); + + private readonly GitlabApi _sut; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + + public GitlabApiTests() + { + _sut = new GitlabApi(_mockGitlabClient.Object, GITLAB_SERVER_URL, _mockOctoLogger.Object); + } + + [Fact] + public async Task GetServerVersion_Returns_Server_Version() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + var result = await _sut.GetServerVersion(); + + result.Version.Should().Be(version); + result.Enterprise.Should().BeTrue(); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Enterprise_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Enterprise Edition)"), Times.Once); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Community_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = false + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Community Edition)"), Times.Once); + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs index 574753cd4..ba0dd76e0 100644 --- a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -40,6 +40,8 @@ public async Task Handle(GenerateScriptCommandArgs args) _log.LogInformation("Generating Script..."); + await _gitlabApi.LogServerVersion(); + var script = await GenerateScript(args); if (script.HasValue() && args.Output.HasValue()) diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs index 72240a3ee..c984cc533 100644 --- a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -40,6 +40,8 @@ public async Task Handle(InventoryReportCommandArgs args) _log.LogInformation("Creating inventory report..."); + await _gitlabApi.LogServerVersion(); + var groupPaths = Array.Empty(); if (string.IsNullOrWhiteSpace(args.GitlabGroup)) { diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs index b8dd659ad..cc290d2c8 100644 --- a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -52,6 +52,8 @@ public async Task Handle(MigrateRepoCommandArgs args) ValidateOptions(args); + await _gitlabApi.LogServerVersion(); + var migrationSourceId = ""; if (args.ShouldImportArchive())