diff --git a/src/Octoshift/Models/GitlabProject.cs b/src/Octoshift/Models/GitlabProject.cs new file mode 100644 index 000000000..bee22dd75 --- /dev/null +++ b/src/Octoshift/Models/GitlabProject.cs @@ -0,0 +1,9 @@ +namespace Octoshift.Models; + +public record GitlabProject +{ + public string Id { get; init; } + public string Name { get; init; } + public string Path { get; init; } + public bool Archived { get; init; } +} diff --git a/src/Octoshift/Services/EnvironmentVariableProvider.cs b/src/Octoshift/Services/EnvironmentVariableProvider.cs index e59b47109..878d92734 100644 --- a/src/Octoshift/Services/EnvironmentVariableProvider.cs +++ b/src/Octoshift/Services/EnvironmentVariableProvider.cs @@ -15,6 +15,7 @@ public class EnvironmentVariableProvider private const string AWS_REGION = "AWS_REGION"; private const string BBS_USERNAME = "BBS_USERNAME"; private const string BBS_PASSWORD = "BBS_PASSWORD"; + private const string GITLAB_PAT = "GITLAB_PAT"; private const string SMB_PASSWORD = "SMB_PASSWORD"; private const string GEI_SKIP_STATUS_CHECK = "GEI_SKIP_STATUS_CHECK"; private const string GEI_SKIP_VERSION_CHECK = "GEI_SKIP_VERSION_CHECK"; @@ -57,6 +58,9 @@ public virtual string BbsUsername(bool throwIfNotFound = true) => public virtual string BbsPassword(bool throwIfNotFound = true) => GetSecret(BBS_PASSWORD, throwIfNotFound); + public virtual string GitlabPat(bool throwIfNotFound = true) => + GetSecret(GITLAB_PAT, throwIfNotFound); + public virtual string SmbPassword(bool throwIfNotFound = true) => GetSecret(SMB_PASSWORD, throwIfNotFound); diff --git a/src/Octoshift/Services/GithubApi.cs b/src/Octoshift/Services/GithubApi.cs index 00bf29e4e..7e42442cb 100644 --- a/src/Octoshift/Services/GithubApi.cs +++ b/src/Octoshift/Services/GithubApi.cs @@ -346,6 +346,30 @@ public virtual async Task CreateBbsMigrationSource(string orgId) return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; } + public virtual async Task CreateGitlabMigrationSource(string orgId) + { + var url = $"{_apiUrl}/graphql"; + + var query = "mutation createMigrationSource($name: String!, $url: String!, $ownerId: ID!, $type: MigrationSourceType!)"; + var gql = "createMigrationSource(input: {name: $name, url: $url, ownerId: $ownerId, type: $type}) { migrationSource { id, name, url, type } }"; + + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + name = "GitLab Source", + url = "https://not-used", + ownerId = orgId, + type = "GITLAB" + }, + operationName = "createMigrationSource" + }; + + var data = await _client.PostGraphQLAsync(url, payload); + return (string)data["data"]["createMigrationSource"]["migrationSource"]["id"]; + } + public virtual async Task CreateGhecMigrationSource(string orgId) { var url = $"{_apiUrl}/graphql"; @@ -534,6 +558,23 @@ public virtual async Task StartBbsMigration(string migrationSourceId, st ); } + public virtual async Task StartGitlabMigration(string migrationSourceId, string gitlabRepoUrl, string orgId, string repo, string targetToken, string archiveUrl, string targetRepoVisibility = null) + { + return await StartMigration( + migrationSourceId, + gitlabRepoUrl, // source repository URL + orgId, + repo, + "not-used", // source access token + targetToken, + archiveUrl, + null, // GitLab archive contains both git and metadata — GitHub falls back to gitArchiveUrl + false, // skip releases + targetRepoVisibility, + false // lock source + ); + } + public virtual async Task<(string State, string RepositoryName, int WarningsCount, string FailureReason, string MigrationLogUrl)> GetMigration(string migrationId) { var url = $"{_apiUrl}/graphql"; diff --git a/src/Octoshift/Services/GithubClient.cs b/src/Octoshift/Services/GithubClient.cs index 6d846bc8d..f31e2a9fc 100644 --- a/src/Octoshift/Services/GithubClient.cs +++ b/src/Octoshift/Services/GithubClient.cs @@ -38,7 +38,7 @@ public GithubClient(OctoLogger log, HttpClient httpClient, IVersionProvider vers if (_httpClient != null) { _httpClient.DefaultRequestHeaders.Add("Accept", "application/vnd.github.v3+json"); - _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api"); + _httpClient.DefaultRequestHeaders.Add("GraphQL-Features", "import_api,mannequin_claiming_emu,org_import_api,octoshift_ll_gitlab_self_serve"); _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", personalAccessToken); _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); if (versionProvider?.GetVersionComments() is { } comments) diff --git a/src/Octoshift/Services/GitlabApi.cs b/src/Octoshift/Services/GitlabApi.cs new file mode 100644 index 000000000..bc31a6ec7 --- /dev/null +++ b/src/Octoshift/Services/GitlabApi.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabApi +{ + private readonly GitlabClient _client; + private readonly string _gitlabBaseUrl; + private readonly OctoLogger _log; + + public GitlabApi(GitlabClient client, string gitlabServerUrl, OctoLogger log) + { + _client = client; + _gitlabBaseUrl = gitlabServerUrl?.TrimEnd('/'); + _log = log; + } + + public virtual async Task<(string Version, bool Enterprise)> GetServerVersion() + { + var url = $"{_gitlabBaseUrl}/api/v4/version"; + + var content = await _client.GetAsync(url); + var data = JObject.Parse(content); + + return ((string)data["version"], (bool?)data["enterprise"] ?? false); + } + + public virtual async Task LogServerVersion() + { + var (version, enterprise) = await GetServerVersion(); + if (!string.IsNullOrWhiteSpace(version)) + { + var edition = enterprise ? "Enterprise" : "Community"; + _log?.LogInformation($"GitLab version: {version} ({edition} Edition)"); + } + } + + public virtual async Task StartExport(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; + + var exportResponse = await _client.PostAsync(url, new { }); + var exportData = JObject.Parse(exportResponse); + + return (string)exportData["message"]; + } + + public virtual async Task<(string ExportStatus, string DownloadUrl)> GetExport(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export"; + + var exportResponse = await _client.GetAsync(url); + var exportData = JObject.Parse(exportResponse); + + return ( + (string)exportData["export_status"], + (string)exportData["_links"]?["api_url"] + ); + } + + public virtual async Task DownloadExportArchive(string groupPath, string projectPath, string file) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/export/download"; + + await _client.DownloadToFile(url, file); + } + + public virtual async Task> GetProjects(string groupPath) + { + var encodedGroupPath = Uri.EscapeDataString(groupPath); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}/projects?per_page=100"; + + return await _client.GetAllAsync(url) + .Select(x => ((long)x["id"], (string)x["path"], (string)x["name"], (bool)x["archived"])) + .ToListAsync(); + } + + public virtual async Task<(long Id, string Path, string Name)> GetGroup(string groupPath) + { + var encodedGroupPath = groupPath.EscapeDataString(); + var url = $"{_gitlabBaseUrl}/api/v4/groups/{encodedGroupPath}"; + + var groupResponse = await _client.GetAsync(url); + var groupData = JObject.Parse(groupResponse); + + return ( + (long)groupData["id"], + (string)groupData["full_path"], + (string)groupData["name"] + ); + } + + public virtual async Task> GetGroups() + { + var url = $"{_gitlabBaseUrl}/api/v4/groups?per_page=100"; + + return await _client.GetAllAsync(url) + .Select(x => ((long)x["id"], (string)x["full_path"], (string)x["name"])) + .ToListAsync(); + } + + public virtual async Task GetIsProjectArchived(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}"; + + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); + + return (bool)projectData["archived"]; + } + + public virtual async Task GetRepositoryLatestCommitDate(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/repository/commits?per_page=1"; + + // Empty projects (no Git repo yet) return 404 here; treat as no commits. + var commitsResponse = await _client.GetOrNullForNotFoundAsync(url); + if (commitsResponse is null) + { + return null; + } + + var commitsData = JArray.Parse(commitsResponse); + var lastCommittedDate = (string)commitsData.First?["committed_date"]; + + if (string.IsNullOrWhiteSpace(lastCommittedDate)) + { + return null; + } + + return DateTimeOffset.Parse(lastCommittedDate); + } + + public virtual async Task<(long RepositorySize, long AttachmentsSize)> GetRepositoryAndAttachmentsSize(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}?statistics=true"; + + var projectResponse = await _client.GetAsync(url); + var projectData = JObject.Parse(projectResponse); + var projectStatistics = (JObject)projectData["statistics"]; + + var repositorySize = (long)projectStatistics["repository_size"]; + var attachmentsSize = (long)projectStatistics["uploads_size"]; + + return (repositorySize, attachmentsSize); + } + + public virtual async Task GetMergeRequestCount(string groupPath, string projectPath) + { + var encodedProjectPath = GetEncodedProjectPath(groupPath, projectPath); + var url = $"{_gitlabBaseUrl}/api/v4/projects/{encodedProjectPath}/merge_requests?state=all&per_page=1&page=1"; + + var mrResponse = await _client.GetAsyncHttpResponseMessage(url); + var mrTotal = mrResponse.Headers.GetValues("X-Total").Single(); + + return int.Parse(mrTotal); + } + + private static string GetEncodedProjectPath(string groupPath, string projectPath) + { + var pathWithNamespace = $"{groupPath}/{projectPath}"; + return pathWithNamespace.EscapeDataString(); + } +} diff --git a/src/Octoshift/Services/GitlabClient.cs b/src/Octoshift/Services/GitlabClient.cs new file mode 100644 index 000000000..f2a0462e7 --- /dev/null +++ b/src/Octoshift/Services/GitlabClient.cs @@ -0,0 +1,172 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; +using System.Web; +using Newtonsoft.Json.Linq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; + +namespace OctoshiftCLI.Services; + +public class GitlabClient +{ + private const int DEFAULT_PAGE_SIZE = 100; + private readonly HttpClient _httpClient; + private readonly OctoLogger _log; + private readonly RetryPolicy _retryPolicy; + private readonly FileSystemProvider _fileSystemProvider; + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, string gitlabPat, FileSystemProvider fileSystemProvider) : + this(log, httpClient, versionProvider, retryPolicy, fileSystemProvider) + { + if (_httpClient != null) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", gitlabPat); + } + } + + public GitlabClient(OctoLogger log, HttpClient httpClient, IVersionProvider versionProvider, RetryPolicy retryPolicy, FileSystemProvider fileSystemProvider) + { + _log = log; + _httpClient = httpClient; + _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; + + if (_httpClient != null) + { + _httpClient.DefaultRequestHeaders.Add("accept", "application/json"); + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("OctoshiftCLI", versionProvider?.GetCurrentVersion())); + if (versionProvider?.GetVersionComments() is { } comments) + { + _httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue(comments)); + } + } + } + + public virtual async Task GetAsync(string url) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + return await response.Content.ReadAsStringAsync(); + } + + /// + /// Like , but returns null when the server responds with 404 Not Found + /// instead of throwing. Other HTTP errors are still retried and bubble up as exceptions. + /// + public virtual async Task GetOrNullForNotFoundAsync(string url) + { + try + { + using var response = await _retryPolicy.HttpRetry( + async () => await SendAsync(HttpMethod.Get, url), + ex => ex.StatusCode != HttpStatusCode.NotFound); + return await response.Content.ReadAsStringAsync(); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public virtual async IAsyncEnumerable GetAllAsync(string url) + { + var nextPage = 1; + while (nextPage > 0) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, AddPageParam(url, nextPage))); + var content = await response.Content.ReadAsStringAsync(); + var jArray = JArray.Parse(content); + + foreach (var jToken in jArray) + { + yield return jToken; + } + + nextPage = response.Headers.TryGetValues("X-Next-Page", out var values) + && int.TryParse(values.FirstOrDefault(), out var parsed) + ? parsed + : 0; + } + } + + public virtual async Task GetAsyncHttpResponseMessage(string url) + { + return await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Get, url)); + } + + public virtual async Task PostAsync(string url, object body) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Post, url, body)); + return await response.Content.ReadAsStringAsync(); + } + + public virtual async Task DeleteAsync(string url) + { + using var response = await _retryPolicy.Retry(async () => await SendAsync(HttpMethod.Delete, url)); + return await response.Content.ReadAsStringAsync(); + } + + private async Task SendAsync(HttpMethod httpMethod, string url, object body = null) + { + _log.LogVerbose($"HTTP {httpMethod}: {url}"); + + if (body != null) + { + _log.LogVerbose($"HTTP BODY: {body.ToJson()}"); + } + + using var payload = body?.ToJson().ToStringContent(); + var response = httpMethod.ToString() switch + { + "GET" => await _httpClient.GetAsync(url), + "DELETE" => await _httpClient.DeleteAsync(url), + "POST" => await _httpClient.PostAsync(url, payload), + "PUT" => await _httpClient.PutAsync(url, payload), + "PATCH" => await _httpClient.PatchAsync(url, payload), + _ => throw new ArgumentOutOfRangeException($"{httpMethod} is not supported.") + }; + var content = await response.Content.ReadAsStringAsync(); + _log.LogVerbose($"RESPONSE ({response.StatusCode}): {content}"); + + response.EnsureSuccessStatusCode(); + + return response; + } + + private static string AddPageParam(string url, int page) + { + var uri = new Uri(url); + var path = uri.GetLeftPart(UriPartial.Path); + var queryParams = HttpUtility.ParseQueryString(uri.Query); + + queryParams["page"] = page.ToString(); + if (string.IsNullOrEmpty(queryParams["per_page"])) + { + queryParams["per_page"] = DEFAULT_PAGE_SIZE.ToString(); + } + + return $"{path}?{queryParams}"; + } + + public virtual async Task DownloadToFile(string url, string file) + { + _log.LogVerbose($"HTTP GET: {url}"); + + using var response = await _retryPolicy.Retry(async () => + await _httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead)); + + _log.LogVerbose($"RESPONSE ({response.StatusCode}): "); + + response.EnsureSuccessStatusCode(); + + await using var streamToReadFrom = await response.Content.ReadAsStreamAsync(); + await using var streamToWriteTo = _fileSystemProvider.Open(file, FileMode.Create); + await _fileSystemProvider.CopySourceToTargetStreamAsync(streamToReadFrom, streamToWriteTo); + } +} diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs index 733621b49..2ca0c36ba 100644 --- a/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GithubApiTests.cs @@ -1202,6 +1202,115 @@ mutation startRepositoryMigration( expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); } + [Fact] + public async Task StartGitlabMigration_Sends_Null_Metadata_Url() + { + // Arrange + const string migrationSourceId = "MIGRATION_SOURCE_ID"; + const string sourceRepoUrl = "https://gitlab.com/my-group/my-project"; + const string orgId = "ORG_ID"; + const string url = "https://api.github.com/graphql"; + const string gitArchiveUrl = "GIT_ARCHIVE_URL"; + const string targetToken = "TARGET_TOKEN"; + + const string unusedSourceToken = "not-used"; + + const string query = @" + mutation startRepositoryMigration( + $sourceId: ID!, + $ownerId: ID!, + $sourceRepositoryUrl: URI!, + $repositoryName: String!, + $continueOnError: Boolean!, + $gitArchiveUrl: String, + $metadataArchiveUrl: String, + $accessToken: String!, + $githubPat: String, + $skipReleases: Boolean, + $targetRepoVisibility: String, + $lockSource: Boolean)"; + const string gql = @" + startRepositoryMigration( + input: { + sourceId: $sourceId, + ownerId: $ownerId, + sourceRepositoryUrl: $sourceRepositoryUrl, + repositoryName: $repositoryName, + continueOnError: $continueOnError, + gitArchiveUrl: $gitArchiveUrl, + metadataArchiveUrl: $metadataArchiveUrl, + accessToken: $accessToken, + githubPat: $githubPat, + skipReleases: $skipReleases, + targetRepoVisibility: $targetRepoVisibility, + lockSource: $lockSource + } + ) { + repositoryMigration { + id, + databaseId, + migrationSource { + id, + name, + type + }, + sourceUrl, + state, + failureReason + } + }"; + var payload = new + { + query = $"{query} {{ {gql} }}", + variables = new + { + sourceId = migrationSourceId, + ownerId = orgId, + sourceRepositoryUrl = sourceRepoUrl, + repositoryName = GITHUB_REPO, + continueOnError = true, + gitArchiveUrl, + metadataArchiveUrl = (string)null, + accessToken = unusedSourceToken, + githubPat = targetToken, + skipReleases = false, + targetRepoVisibility = (string)null, + lockSource = false + }, + operationName = "startRepositoryMigration" + }; + const string actualRepositoryMigrationId = "RM_kgC4NjFhNmE2NGU2ZWE1YTQwMDA5ODliZjhi"; + var response = JObject.Parse($@" + {{ + ""data"": {{ + ""startRepositoryMigration"": {{ + ""repositoryMigration"": {{ + ""id"": ""{actualRepositoryMigrationId}"", + ""databaseId"": ""3ba25b34-b23d-43fb-a819-f44414be8dc0"", + ""migrationSource"": {{ + ""id"": ""MS_kgC4NjFhNmE2NDViNWZmOTEwMDA5MTZiMGQw"", + ""name"": ""GitLab Source"", + ""type"": ""GITLAB"" + }}, + ""sourceUrl"": ""{sourceRepoUrl}"", + ""state"": ""QUEUED"", + ""failureReason"": """" + }} + }} + }} + }}"); + + _githubClientMock + .Setup(m => m.PostGraphQLAsync(url, It.Is(x => x.ToJson() == payload.ToJson()), null)) + .ReturnsAsync(response); + + // Act + var expectedRepositoryMigrationId = await _githubApi.StartGitlabMigration(migrationSourceId, sourceRepoUrl, orgId, GITHUB_REPO, targetToken, gitArchiveUrl); + + // Assert + expectedRepositoryMigrationId.Should().Be(actualRepositoryMigrationId); + } + [Fact] public async Task StartMigration_Does_Not_Throw_When_Errors_Is_Empty() { diff --git a/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs new file mode 100644 index 000000000..badc153b8 --- /dev/null +++ b/src/OctoshiftCLI.Tests/Octoshift/Services/GitlabApiTests.cs @@ -0,0 +1,84 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.Octoshift.Services; + +public class GitlabApiTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabClient = TestHelpers.CreateMock(); + + private readonly GitlabApi _sut; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + + public GitlabApiTests() + { + _sut = new GitlabApi(_mockGitlabClient.Object, GITLAB_SERVER_URL, _mockOctoLogger.Object); + } + + [Fact] + public async Task GetServerVersion_Returns_Server_Version() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + var result = await _sut.GetServerVersion(); + + result.Version.Should().Be(version); + result.Enterprise.Should().BeTrue(); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Enterprise_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0-ee"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = true + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Enterprise Edition)"), Times.Once); + } + + [Fact] + public async Task LogServerVersion_Logs_Version_With_Community_Edition() + { + var endpoint = $"{GITLAB_SERVER_URL}/api/v4/version"; + var version = "18.11.0"; + + var responsePayload = new + { + version, + revision = "abc123", + enterprise = false + }; + + _mockGitlabClient.Setup(x => x.GetAsync(endpoint)).ReturnsAsync(responsePayload.ToJson()); + + await _sut.LogServerVersion(); + + _mockOctoLogger.Verify(m => m.LogInformation($"GitLab version: {version} (Community Edition)"), Times.Once); + } +} diff --git a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj index 833ba590f..835eeefa4 100644 --- a/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj +++ b/src/OctoshiftCLI.Tests/OctoshiftCLI.Tests.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs new file mode 100644 index 000000000..1464e8562 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/AbortMigration/AbortMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.AbortMigration; + +public class AbortMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new AbortMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("abort-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs new file mode 100644 index 000000000..a9a32e501 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/CreateTeam/CreateTeamCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.CreateTeam; + +public class CreateTeamCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new CreateTeamCommand(); + Assert.NotNull(command); + Assert.Equal("create-team", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "team-name", true); + TestHelpers.VerifyCommandOption(command.Options, "idp-group", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs new file mode 100644 index 000000000..6c2a3073a --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/DownloadLogs/DownloadLogsCommandTests.cs @@ -0,0 +1,38 @@ +using System.Linq; +using OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.DownloadLogs; + +public class DownloadLogsCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new DownloadLogsCommand(); + Assert.NotNull(command); + Assert.Equal("download-logs", command.Name); + Assert.Equal(8, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "migration-log-file", false); + TestHelpers.VerifyCommandOption(command.Options, "overwrite", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } + + [Fact] + public void Should_Support_Github_Api_Url_Alias_For_Backward_Compatibility() + { + // Test that --github-api-url still works as an alias for --target-api-url + var command = new DownloadLogsCommand(); + var option = command.Options.FirstOrDefault(o => o.Name == "target-api-url"); + + Assert.NotNull(option); + Assert.Contains("--target-api-url", option.Aliases); + Assert.Contains("--github-api-url", option.Aliases); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs new file mode 100644 index 000000000..c7b663412 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommandTests.cs @@ -0,0 +1,23 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateMannequinCsv; + +public class GenerateMannequinCsvCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GenerateMannequinCsvCommand(); + Assert.NotNull(command); + Assert.Equal("generate-mannequin-csv", command.Name); + Assert.Equal(6, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "output", false); + TestHelpers.VerifyCommandOption(command.Options, "include-reclaimed", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs new file mode 100644 index 000000000..9e617347e --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgsTests.cs @@ -0,0 +1,41 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgsTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly GenerateScriptCommandArgs _args = new(); + + [Fact] + public void It_Throws_If_Both_AwsBucketName_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsBucketName = "my-bucket"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + [Fact] + public void It_Throws_If_Both_AwsRegion_And_UseGithubStorage_Are_Provided() + { + // Arrange + _args.AwsRegion = "aws-region"; + _args.UseGithubStorage = true; + + // Act & Assert + _args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs new file mode 100644 index 000000000..ffabddc50 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandlerTests.cs @@ -0,0 +1,208 @@ +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandlerTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + + private readonly GenerateScriptCommandHandler _handler; + + private const string GITHUB_ORG = "GITHUB-ORG"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string OUTPUT = "unit-test-output"; + private const string GROUP_PATH_FOO = "group-foo"; + private const string GROUP_NAME_FOO = "Group Foo"; + private const string GROUP_PATH_BAR = "group-bar"; + private const string GROUP_NAME_BAR = "Group Bar"; + private const string PROJECT_PATH_1 = "project-1"; + private const string PROJECT_NAME_1 = "Project 1"; + private const string PROJECT_PATH_2 = "project-2"; + private const string PROJECT_NAME_2 = "Project 2"; + private const string AWS_BUCKET_NAME = "AWS-BUCKET-NAME"; + private const string AWS_REGION = "us-east-1"; + + public GenerateScriptCommandHandlerTests() + { + _handler = new GenerateScriptCommandHandler( + _mockOctoLogger.Object, + _mockVersionProvider.Object, + _mockFileSystemProvider.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object); + } + + [Fact] + public async Task No_Output_Path_Does_Not_Write_File() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task No_Groups_Generates_Header_Only() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT) + }; + + await _handler.Handle(args); + + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain("VALIDATE_GH_PAT".Replace("VALIDATE_GH_PAT", "GH_PAT")); + capturedScript.Should().Contain("GITLAB_PAT"); + capturedScript.Should().NotContain("# =========== Group:"); + } + + [Fact] + public async Task Default_Generates_Migrate_Repo_Command_For_Each_Project() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] + { + (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO), + (Id: 2L, Path: GROUP_PATH_BAR, Name: GROUP_NAME_BAR) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false), + (Id: 2L, Path: PROJECT_PATH_2, Name: PROJECT_NAME_2, Archived: false) + }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_BAR)) + .ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name, bool Archived)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT) + }; + + await _handler.Handle(args); + + capturedScript.Should().NotBeNullOrEmpty(); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_FOO} ==========="); + capturedScript.Should().Contain($"# =========== Group: {GROUP_PATH_BAR} ==========="); + capturedScript.Should().Contain($"--gitlab-server-url \"{GITLAB_SERVER_URL}\""); + capturedScript.Should().Contain($"--gitlab-group \"{GROUP_PATH_FOO}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_1}\""); + capturedScript.Should().Contain($"--gitlab-project \"{PROJECT_PATH_2}\""); + capturedScript.Should().Contain($"--github-org \"{GITHUB_ORG}\""); + capturedScript.Should().Contain($"--github-repo \"{GROUP_PATH_FOO}-{PROJECT_PATH_1}\""); + capturedScript.Should().Contain("--target-repo-visibility private"); + capturedScript.Should().Contain("Skipping this group because it has no projects."); + capturedScript.Should().NotContain("--queue-only"); + capturedScript.Should().NotContain("--verbose"); + capturedScript.Should().NotContain("--aws-bucket-name"); + capturedScript.Should().NotContain("--aws-region"); + capturedScript.Should().NotContain("--keep-archive"); + capturedScript.Should().NotContain("--use-github-storage"); + capturedScript.Should().NotContain("--no-ssl-verify"); + capturedScript.Should().NotContain("--kerberos"); + } + + [Fact] + public async Task Includes_Optional_Flags_When_Set() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] { (Id: 1L, Path: GROUP_PATH_FOO, Name: GROUP_NAME_FOO) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_FOO)) + .ReturnsAsync(new[] { (Id: 1L, Path: PROJECT_PATH_1, Name: PROJECT_NAME_1, Archived: false) }); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT), + Verbose = true, + AwsBucketName = AWS_BUCKET_NAME, + AwsRegion = AWS_REGION, + KeepArchive = true + }; + + await _handler.Handle(args); + + capturedScript.Should().Contain("--verbose"); + capturedScript.Should().Contain($"--aws-bucket-name \"{AWS_BUCKET_NAME}\""); + capturedScript.Should().Contain($"--aws-region \"{AWS_REGION}\""); + capturedScript.Should().Contain("--keep-archive"); + capturedScript.Should().Contain("VALIDATE_AWS_ACCESS_KEY_ID".Replace("VALIDATE_", "").Replace("ID", "ID")); + capturedScript.Should().Contain("AWS_SECRET_ACCESS_KEY"); + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); + } + + [Fact] + public async Task UseGithubStorage_Skips_Azure_And_Aws_Validation() + { + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(System.Array.Empty<(long Id, string Path, string Name)>()); + + string capturedScript = null; + _mockFileSystemProvider + .Setup(m => m.WriteAllTextAsync(It.IsAny(), It.IsAny())) + .Callback((_, contents) => capturedScript = contents) + .Returns(Task.CompletedTask); + + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GithubOrg = GITHUB_ORG, + Output = new FileInfo(OUTPUT), + UseGithubStorage = true + }; + + await _handler.Handle(args); + + capturedScript.Should().NotContain("AZURE_STORAGE_CONNECTION_STRING"); + capturedScript.Should().NotContain("AWS_ACCESS_KEY_ID"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs new file mode 100644 index 000000000..231f7e2eb --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GenerateScript/GenerateScriptCommandTests.cs @@ -0,0 +1,71 @@ +using System; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockVersionProvider = new(); + + private readonly GenerateScriptCommand _command = []; + + public GenerateScriptCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IVersionProvider))).Returns(_mockVersionProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + } + + [Fact] + public void Should_Have_Options() + { + _command.Should().NotBeNull(); + _command.Name.Should().Be("generate-script"); + _command.Options.Count.Should().Be(14); + + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(_command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(_command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-project", false); + TestHelpers.VerifyCommandOption(_command.Options, "output", false); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(_command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(_command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(_command.Options, "use-github-storage", false, true); + } + + [Fact] + public void It_Creates_The_GitlabApi_With_The_Provided_Server_Url_And_Pat() + { + var args = new GenerateScriptCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT + }; + + _command.BuildHandler(args, _mockServiceProvider.Object); + + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs new file mode 100644 index 000000000..f61e6e528 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.GrantMigratorRole; + +public class GrantMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new GrantMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("grant-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs new file mode 100644 index 000000000..ab29c15d5 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandHandlerTests.cs @@ -0,0 +1,130 @@ +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandlerTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "foo-group"; + private const string GITLAB_PAT = "gitlab-pat"; + private const bool NO_SSL_VERIFY = true; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGenerator = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGenerator = TestHelpers.CreateMock(); + + private string _groupsCsvOutput = ""; + private string _projectsCsvOutput = ""; + + private readonly InventoryReportCommandHandler _handler; + + public InventoryReportCommandHandlerTests() + { + _handler = new InventoryReportCommandHandler( + TestHelpers.CreateMock().Object, + _mockGitlabApi.Object, + _mockGitlabInspectorService.Object, + _mockGroupsCsvGenerator.Object, + _mockProjectsCsvGenerator.Object) + { + WriteToFile = (path, contents) => + { + if (path == "groups.csv") + { + _groupsCsvOutput = contents; + } + + if (path == "projects.csv") + { + _projectsCsvOutput = contents; + } + + return Task.CompletedTask; + } + }; + } + + [Fact] + public async Task Happy_Path() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); + + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, false)).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + } + + [Fact] + public async Task Scoped_To_Single_Group() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(GITLAB_GROUP)).ReturnsAsync(1); + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GITLAB_GROUP, false)).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + + _mockGitlabApi.Verify(m => m.GetGroups(), Times.Never); + } + + [Fact] + public async Task It_Generates_Minimal_Csvs_When_Requested() + { + var expectedGroupsCsv = "groups csv stuff"; + var expectedProjectsCsv = "projects csv stuff"; + + _mockGitlabApi.Setup(m => m.GetGroups()).ReturnsAsync(new[] { (Id: 1L, Path: GITLAB_GROUP, Name: "Foo Group") }); + _mockGitlabInspectorService.Setup(m => m.GetProjectCount(It.IsAny())).ReturnsAsync(1); + + _mockGroupsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedGroupsCsv); + _mockProjectsCsvGenerator.Setup(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, It.IsAny())).ReturnsAsync(expectedProjectsCsv); + + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = NO_SSL_VERIFY, + Minimal = true + }; + await _handler.Handle(args); + + _groupsCsvOutput.Should().Be(expectedGroupsCsv); + _projectsCsvOutput.Should().Be(expectedProjectsCsv); + + _mockGroupsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); + _mockProjectsCsvGenerator.Verify(m => m.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, null, true)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs new file mode 100644 index 000000000..12e738abf --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/InventoryReport/InventoryReportCommandTests.cs @@ -0,0 +1,67 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandTests + { + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockGroupsCsvGeneratorService = TestHelpers.CreateMock(); + private readonly Mock _mockProjectsCsvGeneratorService = TestHelpers.CreateMock(); + + private readonly ServiceProvider _serviceProvider; + private readonly InventoryReportCommand _command = []; + + public InventoryReportCommandTests() + { + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(_mockOctoLogger.Object) + .AddSingleton(_mockGitlabApiFactory.Object) + .AddSingleton(_mockGitlabInspectorServiceFactory.Object) + .AddSingleton(_mockGroupsCsvGeneratorService.Object) + .AddSingleton(_mockProjectsCsvGeneratorService.Object); + + _serviceProvider = serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Should_Have_Options() + { + _command.Should().NotBeNull(); + _command.Name.Should().Be("inventory-report"); + _command.Options.Count.Should().Be(6); + + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-group", false); + TestHelpers.VerifyCommandOption(_command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(_command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(_command.Options, "minimal", false); + TestHelpers.VerifyCommandOption(_command.Options, "verbose", false); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + var args = new InventoryReportCommandArgs + { + GitlabServerUrl = "https://gitlab.contoso.com", + GitlabPat = "gitlab-pat" + }; + + var handler = _command.BuildHandler(args, _serviceProvider); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify)); + _mockGitlabInspectorServiceFactory.Verify(m => m.Create(It.IsAny())); + } + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs new file mode 100644 index 000000000..ce94458a6 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgsTests.cs @@ -0,0 +1,269 @@ +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgsTests +{ + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + + private const string ARCHIVE_PATH = "path/to/archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_SESSION_TOKEN = "aws-session-token"; + private const string AWS_REGION = "aws-region"; + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PAT = "gitlab-pat"; + + [Fact] + public void It_Throws_When_Neither_Gitlab_Server_Url_Nor_Archive_Source_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-server-url*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Both_Archive_Path_And_Archive_Url_Are_Provided() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Gitlab_Group_Or_Project_Is_Missing_When_Generating_Archive() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-group*--gitlab-project*"); + } + + [Fact] + public void It_Throws_When_Gitlab_Pat_Is_Provided_With_Archive_Path() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GitlabPat = GITLAB_PAT + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--gitlab-pat*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_No_Ssl_Verify_Is_Provided_With_Archive_Url() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + NoSslVerify = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--no-ssl-verify*--archive-path*--archive-url*"); + } + + [Fact] + public void It_Throws_When_Aws_Access_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsAccessKey = AWS_ACCESS_KEY_ID + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Secret_Key_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSecretKey = AWS_SECRET_ACCESS_KEY + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Session_Token_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsSessionToken = AWS_SESSION_TOKEN + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Aws_Region_Provided_Without_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + AwsRegion = AWS_REGION + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*AWS S3*--aws-bucket-name*"); + } + + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Aws_Bucket_Name() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AwsBucketName = AWS_BUCKET_NAME, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with an AWS S3 Bucket name*"); + } + + [Fact] + public void It_Throws_When_Use_Github_Storage_Provided_With_Azure_Storage_Connection_String() + { + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + UseGithubStorage = true + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--use-github-storage flag was provided with a connection string*"); + } + + [Fact] + public void It_Throws_When_Github_Org_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubRepo = GITHUB_REPO + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-org*GitLab archive*"); + } + + [Fact] + public void It_Throws_When_Github_Repo_Is_Missing_For_Import() + { + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)) + .Should() + .ThrowExactly() + .WithMessage("*--github-repo*GitLab archive*"); + } + + [Fact] + public void Valid_Generate_And_Upload_Args_Do_Not_Throw() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + GitlabPat = GITLAB_PAT, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + args.Invoking(x => x.Validate(_mockOctoLogger.Object)).Should().NotThrow(); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs new file mode 100644 index 000000000..399b8d114 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandlerTests.cs @@ -0,0 +1,310 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandlerTests +{ + private readonly Mock _mockGithubApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApi = TestHelpers.CreateMock(); + private readonly Mock _mockAwsApi = TestHelpers.CreateMock(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockHttpDownloadService = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + + private readonly WarningsCountLogger _warningsCountLogger; + private readonly MigrateRepoCommandHandler _handler; + + private const string ARCHIVE_PATH = "/tmp/gitlab-archive.tar"; + private const string ARCHIVE_URL = "https://archive-url/gitlab-archive.tar"; + private const string GITHUB_ORG = "target-org"; + private const string GITHUB_REPO = "target-repo"; + private const string GITHUB_PAT = "github-pat"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + private const string AWS_BUCKET_NAME = "aws-bucket-name"; + private const string AWS_ACCESS_KEY_ID = "aws-access-key-id"; + private const string AWS_SECRET_ACCESS_KEY = "aws-secret-access-key"; + private const string AWS_REGION = "eu-west-1"; + + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const string GITLAB_GROUP = "gitlab-group"; + private const string GITLAB_PROJECT = "gitlab-project"; + private const string GITLAB_PROJECT_URL = $"{GITLAB_SERVER_URL}/{GITLAB_GROUP}/{GITLAB_PROJECT}"; + private const string UNUSED_REPO_URL = "https://not-used"; + + private const string GITHUB_ORG_ID = "github-org-id"; + private const string MIGRATION_SOURCE_ID = "migration-source-id"; + private const string MIGRATION_ID = "migration-id"; + + public MigrateRepoCommandHandlerTests() + { + _warningsCountLogger = new WarningsCountLogger(_mockOctoLogger.Object); + _handler = new MigrateRepoCommandHandler( + _mockOctoLogger.Object, + _mockGithubApi.Object, + _mockGitlabApi.Object, + _mockEnvironmentVariableProvider.Object, + _mockAzureApi.Object, + _mockAwsApi.Object, + _mockHttpDownloadService.Object, + _mockFileSystemProvider.Object, + _warningsCountLogger + ); + + _mockFileSystemProvider.Setup(m => m.FileExists(It.IsAny())).Returns(true); + _mockFileSystemProvider.Setup(m => m.GetTempFileName()).Returns(ARCHIVE_PATH); + } + + [Fact] + public async Task Throws_If_Args_Is_Null() + { + await FluentActions + .Invoking(() => _handler.Handle(null)) + .Should() + .ThrowExactlyAsync(); + } + + [Fact] + public async Task Throws_If_Target_Repo_Already_Exists() + { + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(true); + + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage($"A repository called {GITHUB_ORG}/{GITHUB_REPO} already exists"); + } + + [Fact] + public async Task Generate_Only_Calls_Start_Export_And_Downloads() + { + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)) + .ReturnsAsync(("finished", ARCHIVE_URL)); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; + + await _handler.Handle(args); + + _mockGitlabApi.Verify(m => m.StartExport(GITLAB_GROUP, GITLAB_PROJECT)); + _mockGitlabApi.Verify(m => m.DownloadExportArchive(GITLAB_GROUP, GITLAB_PROJECT, It.IsAny())); + _mockGithubApi.Verify(m => m.DoesRepoExist(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task Ingest_Only_Starts_Gitlab_Migration_With_Unused_Source_Url() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, UNUSED_REPO_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + + var args = new MigrateRepoCommandArgs + { + ArchiveUrl = ARCHIVE_URL, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + UNUSED_REPO_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } + + [Fact] + public async Task Passes_Gitlab_Project_Url_When_All_Gitlab_Args_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(MIGRATION_SOURCE_ID, GITLAB_PROJECT_URL, GITHUB_ORG_ID, GITHUB_REPO, GITHUB_PAT, ARCHIVE_URL, null)) + .ReturnsAsync(MIGRATION_ID); + _mockGitlabApi.Setup(x => x.GetExport(GITLAB_GROUP, GITLAB_PROJECT)).ReturnsAsync(("finished", ARCHIVE_URL)); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockGithubApi.Verify(m => m.StartGitlabMigration( + MIGRATION_SOURCE_ID, + GITLAB_PROJECT_URL, + GITHUB_ORG_ID, + GITHUB_REPO, + GITHUB_PAT, + ARCHIVE_URL, + null)); + } + + [Fact] + public async Task Uploads_To_Aws_When_Aws_Bucket_Name_Provided() + { + _mockEnvironmentVariableProvider.Setup(m => m.AwsAccessKeyId(It.IsAny())).Returns(AWS_ACCESS_KEY_ID); + _mockEnvironmentVariableProvider.Setup(m => m.AwsSecretAccessKey(It.IsAny())).Returns(AWS_SECRET_ACCESS_KEY); + _mockEnvironmentVariableProvider.Setup(m => m.AwsRegion(It.IsAny())).Returns(AWS_REGION); + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAwsApi.Setup(x => x.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())).ReturnsAsync(ARCHIVE_URL); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AwsBucketName = AWS_BUCKET_NAME, + AwsAccessKey = AWS_ACCESS_KEY_ID, + AwsSecretKey = AWS_SECRET_ACCESS_KEY, + AwsRegion = AWS_REGION, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockAwsApi.Verify(m => m.UploadToBucket(AWS_BUCKET_NAME, ARCHIVE_PATH, It.IsAny())); + } + + [Fact] + public async Task Deletes_Archive_By_Default() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(ARCHIVE_PATH)); + } + + [Fact] + public async Task Keeps_Archive_When_KeepArchive_Set() + { + _mockEnvironmentVariableProvider.Setup(m => m.TargetGithubPersonalAccessToken(It.IsAny())).Returns(GITHUB_PAT); + _mockGithubApi.Setup(x => x.DoesRepoExist(GITHUB_ORG, GITHUB_REPO)).ReturnsAsync(false); + _mockGithubApi.Setup(x => x.GetOrganizationId(GITHUB_ORG)).ReturnsAsync(GITHUB_ORG_ID); + _mockGithubApi.Setup(x => x.CreateGitlabMigrationSource(GITHUB_ORG_ID)).ReturnsAsync(MIGRATION_SOURCE_ID); + _mockGithubApi.Setup(x => x.StartGitlabMigration(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(MIGRATION_ID); + _mockAzureApi.Setup(x => x.UploadToBlob(It.IsAny(), It.IsAny())).ReturnsAsync(new Uri(ARCHIVE_URL)); + _mockFileSystemProvider.Setup(m => m.OpenRead(ARCHIVE_PATH)).Returns(new MemoryStream(new byte[] { 1, 2, 3 })); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + GithubPat = GITHUB_PAT, + QueueOnly = true, + KeepArchive = true, + }; + + await _handler.Handle(args); + + _mockFileSystemProvider.Verify(m => m.DeleteIfExists(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Throws_When_Gitlab_Pat_Not_Provided_For_Generate() + { + _mockEnvironmentVariableProvider.Setup(m => m.GitlabPat(It.IsAny())).Returns((string)null); + + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabGroup = GITLAB_GROUP, + GitlabProject = GITLAB_PROJECT, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*GitLab PAT*GITLAB_PAT*--gitlab-pat*"); + } + + [Fact] + public async Task Throws_When_Archive_Path_Does_Not_Exist() + { + _mockFileSystemProvider.Setup(m => m.FileExists(ARCHIVE_PATH)).Returns(false); + + var args = new MigrateRepoCommandArgs + { + ArchivePath = ARCHIVE_PATH, + GithubOrg = GITHUB_ORG, + GithubRepo = GITHUB_REPO, + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING, + }; + + await FluentActions + .Invoking(() => _handler.Handle(args)) + .Should() + .ThrowExactlyAsync() + .WithMessage("*archive*--archive-path*"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs new file mode 100644 index 000000000..7b433b1dc --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/MigrateRepo/MigrateRepoCommandTests.cs @@ -0,0 +1,172 @@ +using System; +using System.Net.Http; +using FluentAssertions; +using Moq; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Factories; +using OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const string GITHUB_ORG = "github-org"; + private const string GITHUB_PAT = "github-pat"; + private const string AZURE_STORAGE_CONNECTION_STRING = "azure-storage-connection-string"; + + private readonly Mock _mockServiceProvider = new(); + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockFileSystemProvider = TestHelpers.CreateMock(); + private readonly Mock _mockGithubApiFactory = new(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockAzureApiFactory = new(); + private readonly Mock _warningsCountLogger = TestHelpers.CreateMock(); + + private readonly MigrateRepoCommand _command = []; + + public MigrateRepoCommandTests() + { + _mockServiceProvider.Setup(m => m.GetService(typeof(OctoLogger))).Returns(_mockOctoLogger.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(EnvironmentVariableProvider))).Returns(_mockEnvironmentVariableProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(FileSystemProvider))).Returns(_mockFileSystemProvider.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(ITargetGithubApiFactory))).Returns(_mockGithubApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(GitlabApiFactory))).Returns(_mockGitlabApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(IAzureApiFactory))).Returns(_mockAzureApiFactory.Object); + _mockServiceProvider.Setup(m => m.GetService(typeof(HttpDownloadServiceFactory))) + .Returns(new HttpDownloadServiceFactory( + _mockOctoLogger.Object, + new Mock().Object, + _mockFileSystemProvider.Object, + new Mock().Object)); + _mockServiceProvider.Setup(m => m.GetService(typeof(WarningsCountLogger))).Returns(_warningsCountLogger.Object); + } + + [Fact] + public void Should_Have_Options() + { + var command = new MigrateRepoCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("migrate-repo"); + command.Options.Count.Should().Be(23); + + TestHelpers.VerifyCommandOption(command.Options, "gitlab-server-url", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-group", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-project", true); + TestHelpers.VerifyCommandOption(command.Options, "gitlab-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-url", false); + TestHelpers.VerifyCommandOption(command.Options, "archive-path", false); + TestHelpers.VerifyCommandOption(command.Options, "azure-storage-connection-string", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-bucket-name", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-access-key", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-session-token", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-region", false); + TestHelpers.VerifyCommandOption(command.Options, "aws-secret-key", false); + TestHelpers.VerifyCommandOption(command.Options, "github-org", false); + TestHelpers.VerifyCommandOption(command.Options, "github-repo", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "queue-only", false); + TestHelpers.VerifyCommandOption(command.Options, "target-repo-visibility", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "keep-archive", false); + TestHelpers.VerifyCommandOption(command.Options, "no-ssl-verify", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-uploads-url", false, true); + TestHelpers.VerifyCommandOption(command.Options, "use-github-storage", false, true); + } + + [Fact] + public void BuildHandler_Creates_The_Handler() + { + var args = new MigrateRepoCommandArgs(); + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockGitlabApiFactory.Verify(m => m.Create(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + _mockAzureApiFactory.Verify(m => m.Create(It.IsAny()), Times.Never); + } + + [Fact] + public void BuildHandler_Creates_GitHub_Api_When_Github_Org_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(null, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Uses_Target_Api_Url_When_Provided() + { + var targetApiUrl = "https://api.github.com"; + var args = new MigrateRepoCommandArgs + { + GithubOrg = GITHUB_ORG, + GithubPat = GITHUB_PAT, + TargetApiUrl = targetApiUrl + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGithubApiFactory.Verify(m => m.Create(targetApiUrl, null, GITHUB_PAT)); + } + + [Fact] + public void BuildHandler_Creates_Gitlab_Api_When_Gitlab_Server_Url_Is_Provided() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, false)); + } + + [Fact] + public void BuildHandler_Forwards_NoSslVerify_To_Gitlab_Api_Factory() + { + var args = new MigrateRepoCommandArgs + { + GitlabServerUrl = GITLAB_SERVER_URL, + GitlabPat = GITLAB_PAT, + NoSslVerify = true + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockGitlabApiFactory.Verify(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, true)); + } + + [Fact] + public void BuildHandler_Creates_Azure_Api_When_Connection_String_Is_Provided_Via_Args() + { + var args = new MigrateRepoCommandArgs + { + AzureStorageConnectionString = AZURE_STORAGE_CONNECTION_STRING + }; + + var handler = _command.BuildHandler(args, _mockServiceProvider.Object); + + handler.Should().NotBeNull(); + _mockAzureApiFactory.Verify(m => m.Create(AZURE_STORAGE_CONNECTION_STRING)); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs new file mode 100644 index 000000000..bdc05d23b --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommandTests.cs @@ -0,0 +1,28 @@ +using OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.ReclaimMannequin; + +public class ReclaimMannequinCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new ReclaimMannequinCommand(); + Assert.NotNull(command); + Assert.Equal("reclaim-mannequin", command.Name); + Assert.Equal(11, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "csv", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-user", false); + TestHelpers.VerifyCommandOption(command.Options, "mannequin-id", false); + TestHelpers.VerifyCommandOption(command.Options, "target-user", false); + TestHelpers.VerifyCommandOption(command.Options, "force", false); + TestHelpers.VerifyCommandOption(command.Options, "no-prompt", false); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "skip-invitation", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs new file mode 100644 index 000000000..de7298f2c --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommandTests.cs @@ -0,0 +1,24 @@ +using OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.RevokeMigratorRole; + +public class RevokeMigratorRoleCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new RevokeMigratorRoleCommand(); + Assert.NotNull(command); + Assert.Equal("revoke-migrator-role", command.Name); + Assert.Equal(7, command.Options.Count); + + TestHelpers.VerifyCommandOption(command.Options, "github-org", true); + TestHelpers.VerifyCommandOption(command.Options, "actor", true); + TestHelpers.VerifyCommandOption(command.Options, "actor-type", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "ghes-api-url", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs new file mode 100644 index 000000000..1c6238b62 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Commands/WaitForMigration/WaitForMigrationCommandTests.cs @@ -0,0 +1,22 @@ +using FluentAssertions; +using OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Commands.WaitForMigration; + +public class WaitForMigrationCommandTests +{ + [Fact] + public void Should_Have_Options() + { + var command = new WaitForMigrationCommand(); + command.Should().NotBeNull(); + command.Name.Should().Be("wait-for-migration"); + command.Options.Count.Should().Be(4); + + TestHelpers.VerifyCommandOption(command.Options, "migration-id", true); + TestHelpers.VerifyCommandOption(command.Options, "github-pat", false); + TestHelpers.VerifyCommandOption(command.Options, "verbose", false); + TestHelpers.VerifyCommandOption(command.Options, "target-api-url", false); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs new file mode 100644 index 000000000..6c8b728c1 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Factories/GitlabApiFactoryTests.cs @@ -0,0 +1,59 @@ +using System.Linq; +using System.Net.Http; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Factories; + +public class GitlabApiFactoryTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + + private readonly Mock _mockOctoLogger = TestHelpers.CreateMock(); + private readonly Mock _mockEnvironmentVariableProvider = TestHelpers.CreateMock(); + private readonly Mock _mockHttpClientFactory = new(); + + private readonly GitlabApiFactory _gitlabApiFactory; + + public GitlabApiFactoryTests() + { + _gitlabApiFactory = new GitlabApiFactory(_mockOctoLogger.Object, _mockHttpClientFactory.Object, _mockEnvironmentVariableProvider.Object, null, null, null); + } + + [Fact] + public void Should_Create_GitlabApi_With_Default() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("Default")) + .Returns(httpClient); + + // Act + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat"); + + // Assert + gitlabApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } + + [Fact] + public void Should_Create_GitlabApi_With_No_Ssl_Verify() + { + using var httpClient = new HttpClient(); + + _mockHttpClientFactory + .Setup(x => x.CreateClient("NoSSL")) + .Returns(httpClient); + + // Act + var gitlabApi = _gitlabApiFactory.Create(GITLAB_SERVER_URL, "pat", true); + + // Assert + gitlabApi.Should().NotBeNull(); + httpClient.DefaultRequestHeaders.Accept.First().MediaType.Should().Be("application/json"); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs new file mode 100644 index 000000000..6df183f3a --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/GitlabInspectorServiceTests.cs @@ -0,0 +1,106 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class GitlabInspectorServiceTests +{ + private readonly OctoLogger _logger = TestHelpers.CreateMock().Object; + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly GitlabInspectorService _service; + + private const string GROUP_PATH_1 = "group-1"; + private const string GROUP_NAME_1 = "Group 1"; + private const string GROUP_PATH_2 = "group-2"; + private const string GROUP_NAME_2 = "Group 2"; + + public GitlabInspectorServiceTests() => _service = new(_logger, _mockGitlabApi.Object); + + [Fact] + public async Task GetGroups_Returns_Path_And_Name() + { + _mockGitlabApi + .Setup(m => m.GetGroups()) + .ReturnsAsync(new[] + { + (Id: 1L, Path: GROUP_PATH_1, Name: GROUP_NAME_1), + (Id: 2L, Path: GROUP_PATH_2, Name: GROUP_NAME_2) + }); + + var result = await _service.GetGroups(); + + result.Should().BeEquivalentTo([(GROUP_PATH_1, GROUP_NAME_1), (GROUP_PATH_2, GROUP_NAME_2)]); + } + + [Fact] + public async Task GetProjects_Returns_Projects_For_Group() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: "project-1", Name: "Project 1", Archived: false), + (Id: 2L, Path: "project-2", Name: "Project 2", Archived: true) + }); + + var result = (await _service.GetProjects(GROUP_PATH_1)).ToList(); + + result.Should().HaveCount(2); + result[0].Path.Should().Be("project-1"); + result[0].Name.Should().Be("Project 1"); + result[1].Path.Should().Be("project-2"); + result[1].Name.Should().Be("Project 2"); + } + + [Fact] + public async Task GetProjectCount_For_Group_Returns_Count() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] + { + (Id: 1L, Path: "p1", Name: "p1", Archived: false), + (Id: 2L, Path: "p2", Name: "p2", Archived: false) + }); + + var result = await _service.GetProjectCount(GROUP_PATH_1); + + result.Should().Be(2); + } + + [Fact] + public async Task GetProjectCount_For_Multiple_Groups_Returns_Sum() + { + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_1)) + .ReturnsAsync(new[] { (Id: 1L, Path: "p1", Name: "p1", Archived: false) }); + _mockGitlabApi + .Setup(m => m.GetProjects(GROUP_PATH_2)) + .ReturnsAsync(new[] + { + (Id: 2L, Path: "p2", Name: "p2", Archived: false), + (Id: 3L, Path: "p3", Name: "p3", Archived: false) + }); + + var result = await _service.GetProjectCount(new[] { GROUP_PATH_1, GROUP_PATH_2 }); + + result.Should().Be(3); + } + + [Fact] + public async Task GetProjectMergeRequestCount_Returns_Count_From_Api() + { + _mockGitlabApi + .Setup(m => m.GetMergeRequestCount(GROUP_PATH_1, "project-1")) + .ReturnsAsync(7); + + var result = await _service.GetProjectMergeRequestCount(GROUP_PATH_1, "project-1"); + + result.Should().Be(7); + } +} diff --git a/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs new file mode 100644 index 000000000..232d7ef79 --- /dev/null +++ b/src/OctoshiftCLI.Tests/gl2gh/Services/ProjectsCsvGeneratorServiceTests.cs @@ -0,0 +1,89 @@ +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using Octoshift.Models; +using OctoshiftCLI.GitlabToGithub; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Services; +using Xunit; + +namespace OctoshiftCLI.Tests.GitlabToGithub.Services; + +public class ProjectsCsvGeneratorServiceTests +{ + private const string GITLAB_SERVER_URL = "https://gitlab.contoso.com"; + private const string GITLAB_PAT = "gitlab-pat"; + private const bool NO_SSL_VERIFY = true; + + private const string GROUP_PATH = "group-1"; + private const string GROUP_NAME = "Group 1"; + private const string PROJECT_PATH = "project-1"; + private const string PROJECT_NAME = "Project 1"; + + private const string FULL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived,mr-count"; + private const string MINIMAL_CSV_HEADER = "group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"; + + private readonly Mock _mockGitlabApi = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabApiFactory = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorService = TestHelpers.CreateMock(); + private readonly Mock _mockGitlabInspectorServiceFactory = TestHelpers.CreateMock(); + + private readonly ProjectsCsvGeneratorService _service; + + public ProjectsCsvGeneratorServiceTests() + { + _mockGitlabApiFactory.Setup(m => m.Create(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY)).Returns(_mockGitlabApi.Object); + _mockGitlabInspectorServiceFactory.Setup(m => m.Create(_mockGitlabApi.Object)).Returns(_mockGitlabInspectorService.Object); + _service = new ProjectsCsvGeneratorService(_mockGitlabInspectorServiceFactory.Object, _mockGitlabApiFactory.Object); + } + + [Fact] + public async Task Generate_Returns_Csv_For_Single_Group() + { + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + const long repoSize = 1234; + const long attachmentsSize = 5678; + const int mrCount = 7; + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + _mockGitlabInspectorService.Setup(m => m.GetProjectMergeRequestCount(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(mrCount); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH); + + var expected = + $"{FULL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\",{mrCount}{Environment.NewLine}"; + + result.Should().Be(expected); + } + + [Fact] + public async Task Generate_Returns_Minimal_Csv_When_Requested() + { + const long repoSize = 1234; + const long attachmentsSize = 5678; + var lastCommitDate = new DateTimeOffset(2024, 1, 2, 3, 4, 5, TimeSpan.Zero); + + _mockGitlabInspectorService.Setup(m => m.GetGroup(GROUP_PATH)).ReturnsAsync((GROUP_PATH, GROUP_NAME)); + _mockGitlabInspectorService + .Setup(m => m.GetProjects(GROUP_PATH)) + .ReturnsAsync(new[] { new GitlabProject { Name = PROJECT_NAME, Path = PROJECT_PATH } }); + _mockGitlabApi.Setup(m => m.GetRepositoryLatestCommitDate(GROUP_PATH, PROJECT_PATH)).ReturnsAsync(lastCommitDate); + _mockGitlabApi.Setup(m => m.GetRepositoryAndAttachmentsSize(GROUP_PATH, PROJECT_PATH)).ReturnsAsync((repoSize, attachmentsSize)); + + var result = await _service.Generate(GITLAB_SERVER_URL, GITLAB_PAT, NO_SSL_VERIFY, GROUP_PATH, minimal: true); + + var expected = + $"{MINIMAL_CSV_HEADER}{Environment.NewLine}" + + $"\"{GROUP_PATH}\",\"{GROUP_NAME}\",\"{PROJECT_NAME}\",\"{GITLAB_SERVER_URL}/{GROUP_PATH}/{PROJECT_PATH}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"False\"{Environment.NewLine}"; + + result.Should().Be(expected); + _mockGitlabInspectorService.Verify(m => m.GetProjectMergeRequestCount(It.IsAny(), It.IsAny()), Times.Never); + } +} diff --git a/src/OctoshiftCLI.sln b/src/OctoshiftCLI.sln index 72aee8119..17c0dcb4b 100644 --- a/src/OctoshiftCLI.sln +++ b/src/OctoshiftCLI.sln @@ -21,6 +21,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OctoshiftCLI.IntegrationTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "bbs2gh", "bbs2gh\bbs2gh.csproj", "{39EE734C-AC4A-42AC-801A-ECCA661C23A1}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "gl2gh", "gl2gh\gl2gh.csproj", "{180D1E3B-CA23-4603-BE00-9C856E0665CF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +53,10 @@ Global {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Debug|Any CPU.Build.0 = Debug|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.ActiveCfg = Release|Any CPU {39EE734C-AC4A-42AC-801A-ECCA661C23A1}.Release|Any CPU.Build.0 = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {180D1E3B-CA23-4603-BE00-9C856E0665CF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs new file mode 100644 index 000000000..06fc255ec --- /dev/null +++ b/src/gl2gh/Commands/AbortMigration/AbortMigrationCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.AbortMigration; + +namespace OctoshiftCLI.GitlabToGithub.Commands.AbortMigration; + +public sealed class AbortMigrationCommand : AbortMigrationCommandBase +{ + public AbortMigrationCommand() : base() => AddOptions(); +} diff --git a/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs new file mode 100644 index 000000000..56911ff57 --- /dev/null +++ b/src/gl2gh/Commands/CreateTeam/CreateTeamCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.CreateTeam; + +namespace OctoshiftCLI.GitlabToGithub.Commands.CreateTeam; + +public sealed class CreateTeamCommand : CreateTeamCommandBase +{ + public CreateTeamCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs new file mode 100644 index 000000000..52adb4ce4 --- /dev/null +++ b/src/gl2gh/Commands/DownloadLogs/DownloadLogsCommand.cs @@ -0,0 +1,17 @@ +using System.Runtime.CompilerServices; +using OctoshiftCLI.Commands.DownloadLogs; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] + +namespace OctoshiftCLI.GitlabToGithub.Commands.DownloadLogs; + +public sealed class DownloadLogsCommand : DownloadLogsCommandBase +{ + public DownloadLogsCommand() + { + // Add backward compatibility alias for --github-api-url + GithubApiUrl.AddAlias("--github-api-url"); + + AddOptions(); + } +} diff --git a/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs new file mode 100644 index 000000000..7632ec845 --- /dev/null +++ b/src/gl2gh/Commands/GenerateMannequinCsv/GenerateMannequinCsvCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GenerateMannequinCsv; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateMannequinCsv; + +public sealed class GenerateMannequinCsvCommand : GenerateMannequinCsvCommandBase +{ + public GenerateMannequinCsvCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs new file mode 100644 index 000000000..79820f36a --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommand.cs @@ -0,0 +1,119 @@ +using System; +using System.CommandLine; +using System.IO; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommand : CommandBase +{ + public GenerateScriptCommand() : base( + name: "generate-script", + description: "Generates a migration script. This provides you the ability to review the steps that this tool will take, and optionally modify the script if desired before running it.") + { + AddOption(GitlabServerUrl); + AddOption(GithubOrg); + AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); + AddOption(GitlabPat); + AddOption(GitlabGroup); + AddOption(GitlabProject); + AddOption(Output); + AddOption(Verbose); + AddOption(AwsBucketName); + AddOption(AwsRegion); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server to migrate from, e.g. https://gitlab.mycompany.com") + { IsRequired = true }; + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT of a user with admin privileges to get the list of all groups and their projects. If not set will be read from GITLAB_PAT environment variable." + + $"{Environment.NewLine}" + + "Note: The PAT will not get included in the generated script and it has to be set as an env variable before running the script."); + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group to migrate. If not set will migrate all groups the user has access to."); + + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate. Requires --gitlab-group. If not set will migrate all projects in the group."); + + public Option GithubOrg { get; } = new("--github-org") + { IsRequired = true }; + + public Option Output { get; } = new( + name: "--output", + getDefaultValue: () => new FileInfo("./migrate.ps1")); + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); + + public Option UseGithubStorage { get; } = new("--use-github-storage") + { + IsHidden = true, + Description = "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).", + }; + + public override GenerateScriptCommandHandler BuildHandler(GenerateScriptCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var versionProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + var environmentVariableProvider = sp.GetRequiredService(); + + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + + return new GenerateScriptCommandHandler(log, versionProvider, fileSystemProvider, gitlabApi, environmentVariableProvider); + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs new file mode 100644 index 000000000..c03cbdb6f --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandArgs.cs @@ -0,0 +1,42 @@ +using System.IO; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandArgs : CommandArgs +{ + public string GitlabServerUrl { get; set; } + public string GithubOrg { get; set; } + [Secret] + public string GitlabPat { get; set; } + public string GitlabGroup { get; set; } + public string GitlabProject { get; set; } + public bool NoSslVerify { get; set; } + public FileInfo Output { get; set; } + public string AwsBucketName { get; set; } + public string AwsRegion { get; set; } + public bool KeepArchive { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (GitlabProject.HasValue() && GitlabGroup.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--gitlab-group must be provided when --gitlab-project is specified."); + } + + if (AwsBucketName.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + + if (AwsRegion.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 region. Archive cannot be uploaded to both locations."); + } + } +} diff --git a/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs new file mode 100644 index 000000000..ba0dd76e0 --- /dev/null +++ b/src/gl2gh/Commands/GenerateScript/GenerateScriptCommandHandler.cs @@ -0,0 +1,187 @@ +using System; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GenerateScript; + +public class GenerateScriptCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly IVersionProvider _versionProvider; + private readonly FileSystemProvider _fileSystemProvider; + private readonly GitlabApi _gitlabApi; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + + public GenerateScriptCommandHandler( + OctoLogger log, + IVersionProvider versionProvider, + FileSystemProvider fileSystemProvider, + GitlabApi gitlabApi, + EnvironmentVariableProvider environmentVariableProvider) + { + _log = log; + _versionProvider = versionProvider; + _fileSystemProvider = fileSystemProvider; + _gitlabApi = gitlabApi; + _environmentVariableProvider = environmentVariableProvider; + } + + public async Task Handle(GenerateScriptCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Generating Script..."); + + await _gitlabApi.LogServerVersion(); + + var script = await GenerateScript(args); + + if (script.HasValue() && args.Output.HasValue()) + { + await _fileSystemProvider.WriteAllTextAsync(args.Output.FullName, script); + } + } + + private async Task GenerateScript(GenerateScriptCommandArgs args) + { + var content = new StringBuilder(); + content.AppendLine(PWSH_SHEBANG); + content.AppendLine(); + content.AppendLine(VersionComment); + content.AppendLine(EXEC_FUNCTION_BLOCK); + + content.AppendLine(VALIDATE_GH_PAT); + content.AppendLine(VALIDATE_GITLAB_PAT); + if (args.AwsBucketName.HasValue() || args.AwsRegion.HasValue()) + { + content.AppendLine(VALIDATE_AWS_ACCESS_KEY_ID); + content.AppendLine(VALIDATE_AWS_SECRET_ACCESS_KEY); + } + else if (!args.UseGithubStorage) + { + content.AppendLine(VALIDATE_AZURE_STORAGE_CONNECTION_STRING); + } + + var groups = args.GitlabGroup.HasValue() + ? new[] { args.GitlabGroup } + : (await _gitlabApi.GetGroups()).Select(x => x.Path); + + foreach (var groupPath in groups) + { + _log.LogInformation($"Group: {groupPath}"); + + content.AppendLine(); + content.AppendLine($"# =========== Group: {groupPath} ==========="); + + var projects = await _gitlabApi.GetProjects(groupPath); + + if (args.GitlabProject.HasValue()) + { + projects = projects.Where(p => p.Path == args.GitlabProject).ToArray(); + } + + if (!projects.Any()) + { + content.AppendLine("# Skipping this group because it has no projects."); + continue; + } + + content.AppendLine(); + + foreach (var (_, projectPath, projectName, _) in projects) + { + _log.LogInformation($" Project: {projectName}"); + + content.AppendLine(Exec(MigrateGithubRepoScript(args, groupPath, projectPath, true))); + } + } + + return content.ToString(); + } + + private string MigrateGithubRepoScript(GenerateScriptCommandArgs args, string gitlabGroup, string gitlabProject, bool wait) + { + var gitlabServerUrlOption = $" --gitlab-server-url \"{args.GitlabServerUrl}\""; + var gitlabGroupOption = $" --gitlab-group \"{gitlabGroup}\""; + var gitlabProjectOption = $" --gitlab-project \"{gitlabProject}\""; + var githubOrgOption = $" --github-org \"{args.GithubOrg}\""; + var githubRepoOption = $" --github-repo \"{GetGithubRepoName(gitlabGroup, gitlabProject)}\""; + var waitOption = wait ? "" : " --queue-only"; + var verboseOption = args.Verbose ? " --verbose" : ""; + var awsBucketNameOption = args.AwsBucketName.HasValue() ? $" --aws-bucket-name \"{args.AwsBucketName}\"" : ""; + var awsRegionOption = args.AwsRegion.HasValue() ? $" --aws-region \"{args.AwsRegion}\"" : ""; + var keepArchive = args.KeepArchive ? " --keep-archive" : ""; + var noSslVerifyOption = args.NoSslVerify ? " --no-ssl-verify" : ""; + var targetRepoVisibility = " --target-repo-visibility private"; + var targetApiUrlOption = args.TargetApiUrl.HasValue() ? $" --target-api-url \"{args.TargetApiUrl}\"" : ""; + var targetUploadsUrlOption = args.TargetUploadsUrl.HasValue() ? $" --target-uploads-url \"{args.TargetUploadsUrl}\"" : ""; + var githubStorageOption = args.UseGithubStorage ? " --use-github-storage" : ""; + + return $"gh gl2gh migrate-repo{targetApiUrlOption}{targetUploadsUrlOption}{gitlabServerUrlOption}{gitlabGroupOption}{gitlabProjectOption}" + + $"{githubOrgOption}{githubRepoOption}{verboseOption}{waitOption}{awsBucketNameOption}{awsRegionOption}{keepArchive}{noSslVerifyOption}{targetRepoVisibility}{githubStorageOption}"; + } + + private string Exec(string script) => Wrap(script, "Exec"); + + private string Wrap(string script, string outerCommand = "") => script.IsNullOrWhiteSpace() ? string.Empty : $"{outerCommand} {{ {script} }}".Trim(); + + private string GetGithubRepoName(string gitlabGroup, string gitlabProject) => $"{gitlabGroup}-{gitlabProject}".ReplaceInvalidCharactersWithDash(); + + private string VersionComment => $"# =========== Created with CLI version {_versionProvider.GetCurrentVersion()} ==========="; + + private const string PWSH_SHEBANG = "#!/usr/bin/env pwsh"; + + private const string EXEC_FUNCTION_BLOCK = @" +function Exec { + param ( + [scriptblock]$ScriptBlock + ) + & @ScriptBlock + if ($lastexitcode -ne 0) { + exit $lastexitcode + } +}"; + private const string VALIDATE_GH_PAT = @" +if (-not $env:GH_PAT) { + Write-Error ""GH_PAT environment variable must be set to a valid GitHub Personal Access Token with the appropriate scopes. For more information see https://docs.github.com/en/migrations/using-github-enterprise-importer/preparing-to-migrate-with-github-enterprise-importer/managing-access-for-github-enterprise-importer#creating-a-personal-access-token-for-github-enterprise-importer"" + exit 1 +} else { + Write-Host ""GH_PAT environment variable is set and will be used to authenticate to GitHub."" +}"; + private const string VALIDATE_GITLAB_PAT = @" +if (-not $env:GITLAB_PAT) { + Write-Error ""GITLAB_PAT environment variable must be set to a valid PAT that will be used to call the GitLab API to generate a migration archive."" + exit 1 +} else { + Write-Host ""GITLAB_PAT environment variable is set and will be used to authenticate to the GitLab API."" +}"; + private const string VALIDATE_AZURE_STORAGE_CONNECTION_STRING = @" +if (-not $env:AZURE_STORAGE_CONNECTION_STRING) { + Write-Error ""AZURE_STORAGE_CONNECTION_STRING environment variable must be set to a valid Azure Storage Connection String that will be used to upload the migration archive to Azure Blob Storage."" + exit 1 +} else { + Write-Host ""AZURE_STORAGE_CONNECTION_STRING environment variable is set and will be used to upload the migration archive to Azure Blob Storage."" +}"; + private const string VALIDATE_AWS_ACCESS_KEY_ID = @" +if (-not $env:AWS_ACCESS_KEY_ID) { + Write-Error ""AWS_ACCESS_KEY_ID environment variable must be set to a valid AWS Access Key ID that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_ACCESS_KEY_ID environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; + private const string VALIDATE_AWS_SECRET_ACCESS_KEY = @" +if (-not $env:AWS_SECRET_ACCESS_KEY) { + Write-Error ""AWS_SECRET_ACCESS_KEY environment variable must be set to a valid AWS Secret Access Key that will be used to upload the migration archive to AWS S3."" + exit 1 +} else { + Write-Host ""AWS_SECRET_ACCESS_KEY environment variable is set and will be used to upload the migration archive to AWS S3."" +}"; +} diff --git a/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs new file mode 100644 index 000000000..fdd3ec62e --- /dev/null +++ b/src/gl2gh/Commands/GrantMigratorRole/GrantMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.GrantMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.GrantMigratorRole; + +public sealed class GrantMigratorRoleCommand : GrantMigratorRoleCommandBase +{ + public GrantMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs new file mode 100644 index 000000000..fc165742a --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommand.cs @@ -0,0 +1,78 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommand : CommandBase + { + public InventoryReportCommand() : base( + name: "inventory-report", + description: "Generates several CSV files containing lists of GitLab groups and projects. Useful for planning large migrations. Personal projects owned by individual users will not be included." + + Environment.NewLine + + "Note: Expects GITLAB_PAT env variable or --gitlab-pat options to be set.") + { + AddOption(GitlabServerUrl); + AddOption(GitlabGroup); + AddOption(GitlabPat); + AddOption(NoSslVerify); + AddOption(Minimal); + AddOption(Verbose); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com") + { IsRequired = true }; + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group. Iterates over all projects that the user has access to if not provided."); + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); + + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. " + + "If your GitLab instance has a self-signed SSL certificate then setting this flag will allow data to be extracted."); + + public Option Minimal { get; } = new( + name: "--minimal", + description: "Omit the MR count from group and project reports for quicker report generation."); + + public Option Verbose { get; } = new("--verbose"); + + public override InventoryReportCommandHandler BuildHandler(InventoryReportCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var gitlabApiFactory = sp.GetRequiredService(); + var gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + var gitlabInspectorServiceFactory = sp.GetRequiredService(); + var gitlabInspectorService = gitlabInspectorServiceFactory.Create(gitlabApi); + var groupsCsvGeneratorService = sp.GetRequiredService(); + var projectsCsvGeneratorService = sp.GetRequiredService(); + + return new InventoryReportCommandHandler( + log, + gitlabApi, + gitlabInspectorService, + groupsCsvGeneratorService, + projectsCsvGeneratorService); + } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs new file mode 100644 index 000000000..e7ba94096 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandArgs.cs @@ -0,0 +1,14 @@ +using OctoshiftCLI.Commands; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport +{ + public class InventoryReportCommandArgs : CommandArgs + { + public string GitlabServerUrl { get; set; } + public string GitlabGroup { get; set; } + [Secret] + public string GitlabPat { get; set; } + public bool NoSslVerify { get; set; } + public bool Minimal { get; set; } + } +} diff --git a/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs new file mode 100644 index 000000000..c984cc533 --- /dev/null +++ b/src/gl2gh/Commands/InventoryReport/InventoryReportCommandHandler.cs @@ -0,0 +1,68 @@ +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.InventoryReport; + +public class InventoryReportCommandHandler : ICommandHandler +{ + internal Func WriteToFile = async (path, contents) => await File.WriteAllTextAsync(path, contents); + + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + private readonly GitlabInspectorService _gitlabInspectorService; + private readonly GroupsCsvGeneratorService _groupsCsvGenerator; + private readonly ProjectsCsvGeneratorService _projectsCsvGenerator; + + public InventoryReportCommandHandler( + OctoLogger log, + GitlabApi gitlabApi, + GitlabInspectorService gitlabInspectorService, + GroupsCsvGeneratorService groupsCsvGeneratorService, + ProjectsCsvGeneratorService projectsCsvGeneratorService) + { + _log = log; + _gitlabApi = gitlabApi; + _gitlabInspectorService = gitlabInspectorService; + _groupsCsvGenerator = groupsCsvGeneratorService; + _projectsCsvGenerator = projectsCsvGeneratorService; + } + + public async Task Handle(InventoryReportCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + _log.LogInformation("Creating inventory report..."); + + await _gitlabApi.LogServerVersion(); + + var groupPaths = Array.Empty(); + if (string.IsNullOrWhiteSpace(args.GitlabGroup)) + { + _log.LogInformation("Finding Groups..."); + var groups = await _gitlabApi.GetGroups(); + groupPaths = groups.Select(x => x.Path).ToArray(); + _log.LogInformation($"Found {groups.Count()} Groups"); + } + + _log.LogInformation("Finding Projects..."); + var projectCount = string.IsNullOrWhiteSpace(args.GitlabGroup) ? await _gitlabInspectorService.GetProjectCount(groupPaths) : await _gitlabInspectorService.GetProjectCount(args.GitlabGroup); + _log.LogInformation($"Found {projectCount} Projects"); + + _log.LogInformation("Generating data for groups.csv..."); + var groupsCsvText = await _groupsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); + await WriteToFile("groups.csv", groupsCsvText); + _log.LogSuccess("groups.csv generated"); + + _log.LogInformation("Generating projects.csv..."); + var projectsCsvText = await _projectsCsvGenerator.Generate(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify, args.GitlabGroup, args.Minimal); + await WriteToFile("projects.csv", projectsCsvText); + _log.LogSuccess("projects.csv generated"); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs new file mode 100644 index 000000000..ac4e29925 --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommand.cs @@ -0,0 +1,197 @@ +using System; +using System.CommandLine; +using Microsoft.Extensions.DependencyInjection; +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommand : CommandBase +{ + public MigrateRepoCommand() : base( + name: "migrate-repo", + description: "Import a GitLab archive to GitHub." + + Environment.NewLine + + "Note: Expects GH_PAT env variable or --github-pat option to be set.") + { + AddOption(ArchiveUrl); + AddOption(GithubOrg); + AddOption(GithubRepo); + AddOption(GithubPat); + AddOption(GitlabServerUrl); + AddOption(GitlabGroup); + AddOption(GitlabProject); + AddOption(GitlabPat); + AddOption(ArchivePath); + AddOption(AzureStorageConnectionString); + AddOption(AwsBucketName); + AddOption(AwsAccessKey); + AddOption(AwsSecretKey); + AddOption(AwsSessionToken); + AddOption(AwsRegion); + AddOption(QueueOnly); + AddOption(TargetRepoVisibility.FromAmong("public", "private", "internal")); + AddOption(Verbose); + AddOption(KeepArchive); + AddOption(NoSslVerify); + AddOption(TargetApiUrl); + AddOption(TargetUploadsUrl); + AddOption(UseGithubStorage); + } + + public Option GitlabServerUrl { get; } = new( + name: "--gitlab-server-url", + description: "The full URL of the GitLab server, e.g. https://gitlab.mycompany.com") + { + IsRequired = true + }; + + public Option GitlabGroup { get; } = new( + name: "--gitlab-group", + description: "The GitLab group (full namespace path) that contains the project to migrate. For nested subgroups, use the full path, e.g. parent-group/subgroup.") + { + IsRequired = true + }; + + public Option GitlabProject { get; } = new( + name: "--gitlab-project", + description: "The GitLab project to migrate.") + { + IsRequired = true + }; + + public Option GitlabPat { get; } = new( + name: "--gitlab-pat", + description: "The GitLab PAT. If not passed, it will read the PAT from the GITLAB_PAT environment variable."); + + public Option ArchiveUrl { get; } = new( + name: "--archive-url", + description: + "URL used to download the GitLab migration archive. Only needed if you want to manually retrieve the archive from GitLab instead of letting this CLI do that for you."); + + public Option ArchivePath { get; } = new( + name: "--archive-path", + description: "Path to the GitLab migration archive on disk. When --gitlab-server-url is provided, the generated archive will be written to this path (overwriting any existing file)."); + + public Option AzureStorageConnectionString { get; } = new( + name: "--azure-storage-connection-string", + description: "A connection string for an Azure Storage account, used to upload the GitLab archive. If not passed, it will read the AZURE_STORAGE_CONNECTION_STRING environment variable."); + + public Option AwsBucketName { get; } = new( + name: "--aws-bucket-name", + description: "If using AWS, the name of the S3 bucket to upload the GitLab archive to."); + + public Option AwsAccessKey { get; } = new( + name: "--aws-access-key", + description: "If uploading to S3, the AWS access key. If not provided, it will be read from AWS_ACCESS_KEY_ID environment variable."); + + public Option AwsSecretKey { get; } = new( + name: "--aws-secret-key", + description: "If uploading to S3, the AWS secret key. If not provided, it will be read from AWS_SECRET_ACCESS_KEY environment variable."); + + public Option AwsSessionToken { get; } = new( + name: "--aws-session-token", + description: "If using AWS, the AWS session token. If not provided, it will be read from AWS_SESSION_TOKEN environment variable."); + + public Option AwsRegion { get; } = new( + name: "--aws-region", + description: "If using AWS, the AWS region. If not provided, it will be read from AWS_REGION environment variable. " + + "Required if using AWS."); + + public Option GithubOrg { get; } = new("--github-org"); + + public Option GithubRepo { get; } = new("--github-repo"); + + public Option GithubPat { get; } = new( + name: "--github-pat", + description: "The GitHub personal access token to be used for the migration. If not set will be read from GH_PAT environment variable."); + + public Option QueueOnly { get; } = new( + name: "--queue-only", + description: "Only queues the migration, does not wait for it to finish. Use the wait-for-migration command to subsequently wait for it to finish and view the status."); + + public Option TargetRepoVisibility { get; } = new( + name: "--target-repo-visibility", + description: "The visibility of the target repo. Defaults to private. Valid values are public, private, or internal."); + + public Option Verbose { get; } = new("--verbose"); + + public Option KeepArchive { get; } = new( + name: "--keep-archive", + description: "Keeps the downloaded export archive after successfully uploading it. By default, it will be automatically deleted."); + public Option TargetApiUrl { get; } = new("--target-api-url") + { + Description = "The URL of the target API, if not migrating to github.com. Defaults to https://api.github.com" + }; + public Option TargetUploadsUrl { get; } = new( + name: "--target-uploads-url", + description: "The URL of the target uploads API, if not migrating to github.com. Defaults to https://uploads.github.com") + { IsHidden = true }; + public Option NoSslVerify { get; } = new( + name: "--no-ssl-verify", + description: "Disables SSL verification when communicating with your GitLab instance. All other migration steps will continue to verify SSL. " + + "If your GitLab instance has a self-signed SSL certificate, this flag will allow the migration archive to be exported."); + public Option UseGithubStorage { get; } = new( + name: "--use-github-storage", + description: "Enables multipart uploads to a GitHub owned storage for use during migration. " + + "Configure chunk size with the GITHUB_OWNED_STORAGE_MULTIPART_MEBIBYTES environment variable (default: 100 MiB, minimum: 5 MiB).") + { IsHidden = true }; + + public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs args, IServiceProvider sp) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + if (sp is null) + { + throw new ArgumentNullException(nameof(sp)); + } + + var log = sp.GetRequiredService(); + var environmentVariableProvider = sp.GetRequiredService(); + var fileSystemProvider = sp.GetRequiredService(); + var httpDownloadServiceFactory = sp.GetRequiredService(); + var httpDownloadService = args.NoSslVerify ? httpDownloadServiceFactory.CreateClientNoSsl() : httpDownloadServiceFactory.CreateDefault(); + + GithubApi githubApi = null; + GitlabApi gitlabApi = null; + AzureApi azureApi = null; + AwsApi awsApi = null; + + if (args.GithubOrg.HasValue()) + { + var githubApiFactory = sp.GetRequiredService(); + githubApi = githubApiFactory.Create(args.TargetApiUrl, args.TargetUploadsUrl, args.GithubPat); + } + + if (args.GitlabServerUrl.HasValue()) + { + var gitlabApiFactory = sp.GetRequiredService(); + + gitlabApi = gitlabApiFactory.Create(args.GitlabServerUrl, args.GitlabPat, args.NoSslVerify); + } + + var azureStorageConnectionString = args.AzureStorageConnectionString ?? environmentVariableProvider.AzureStorageConnectionString(false); + if (azureStorageConnectionString.HasValue()) + { + var azureApiFactory = sp.GetRequiredService(); + azureApi = azureApiFactory.Create(azureStorageConnectionString); + } + + if (args.AwsBucketName.HasValue()) + { + var awsApiFactory = sp.GetRequiredService(); + awsApi = awsApiFactory.Create(args.AwsRegion, args.AwsAccessKey, args.AwsSecretKey, args.AwsSessionToken); + } + + var warningsCountLogger = sp.GetRequiredService(); + + return new MigrateRepoCommandHandler(log, githubApi, gitlabApi, environmentVariableProvider, azureApi, awsApi, httpDownloadService, fileSystemProvider, warningsCountLogger); + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs new file mode 100644 index 000000000..603dfb36e --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandArgs.cs @@ -0,0 +1,132 @@ +using System.Linq; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandArgs : CommandArgs +{ + public string ArchiveUrl { get; set; } + public string ArchivePath { get; set; } + + [Secret] + public string AzureStorageConnectionString { get; set; } + + public string AwsBucketName { get; set; } + [Secret] + public string AwsAccessKey { get; set; } + [Secret] + public string AwsSecretKey { get; set; } + [Secret] + public string AwsSessionToken { get; set; } + public string AwsRegion { get; set; } + + public string GithubOrg { get; set; } + public string GithubRepo { get; set; } + [Secret] + public string GithubPat { get; set; } + public bool QueueOnly { get; set; } + public string TargetRepoVisibility { get; set; } + public string TargetApiUrl { get; set; } + public string TargetUploadsUrl { get; set; } + + public string GitlabServerUrl { get; set; } + public string GitlabGroup { get; set; } + public string GitlabProject { get; set; } + [Secret] + public string GitlabPat { get; set; } + public bool NoSslVerify { get; set; } + + public bool KeepArchive { get; set; } + public bool UseGithubStorage { get; set; } + + public override void Validate(OctoLogger log) + { + if (!GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue() && !ArchivePath.HasValue()) + { + throw new OctoshiftCliException("Either --gitlab-server-url, --archive-path, or --archive-url must be specified."); + } + + if (ArchivePath.HasValue() && ArchiveUrl.HasValue()) + { + throw new OctoshiftCliException("Only one of --archive-path or --archive-url can be specified."); + } + + if (ShouldGenerateArchive()) + { + ValidateGenerateOptions(); + } + else + { + ValidateNoGenerateOptions(); + } + + if (ShouldUploadArchive()) + { + ValidateUploadOptions(); + } + + if (ShouldImportArchive()) + { + ValidateImportOptions(); + } + } + + private void ValidateNoGenerateOptions() + { + if (GitlabPat.HasValue()) + { + throw new OctoshiftCliException("--gitlab-pat cannot be provided with --archive-path or --archive-url."); + } + + if (NoSslVerify) + { + throw new OctoshiftCliException("--no-ssl-verify cannot be provided with --archive-path or --archive-url."); + } + } + + public bool ShouldGenerateArchive() => GitlabServerUrl.HasValue() && !ArchiveUrl.HasValue(); + + public bool ShouldUploadArchive() => ArchiveUrl.IsNullOrWhiteSpace() && GithubOrg.HasValue(); + + // NOTE: ArchiveUrl doesn't necessarily refer to the value passed in by the user to the CLI - it is set during CLI runtime when an archive is uploaded to blob storage + public bool ShouldImportArchive() => ArchiveUrl.HasValue() || GithubOrg.HasValue(); + + private void ValidateGenerateOptions() + { + if (GitlabGroup.IsNullOrWhiteSpace() || GitlabProject.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Both --gitlab-group and --gitlab-project must be provided."); + } + } + + private void ValidateUploadOptions() + { + if (AwsBucketName.IsNullOrWhiteSpace() && new[] { AwsAccessKey, AwsSecretKey, AwsSessionToken, AwsRegion }.Any(x => x.HasValue())) + { + throw new OctoshiftCliException("The AWS S3 bucket name must be provided with --aws-bucket-name if other AWS S3 upload options are set."); + } + if (UseGithubStorage && AwsBucketName.HasValue()) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with an AWS S3 Bucket name. Archive cannot be uploaded to both locations."); + } + if (AzureStorageConnectionString.HasValue() && UseGithubStorage) + { + throw new OctoshiftCliException("The --use-github-storage flag was provided with a connection string for an Azure storage account. Archive cannot be uploaded to both locations."); + } + } + + private void ValidateImportOptions() + { + if (GithubOrg.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-org must be provided in order to import the GitLab archive."); + } + + if (GithubRepo.IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("--github-repo must be provided in order to import the GitLab archive."); + } + } +} diff --git a/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs new file mode 100644 index 000000000..cc290d2c8 --- /dev/null +++ b/src/gl2gh/Commands/MigrateRepo/MigrateRepoCommandHandler.cs @@ -0,0 +1,350 @@ +using System; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using OctoshiftCLI.Commands; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Commands.MigrateRepo; + +public class MigrateRepoCommandHandler : ICommandHandler +{ + private readonly OctoLogger _log; + private readonly GithubApi _githubApi; + private readonly GitlabApi _gitlabApi; + private readonly AzureApi _azureApi; + private readonly AwsApi _awsApi; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly HttpDownloadService _httpDownloadService; + private readonly FileSystemProvider _fileSystemProvider; + private readonly WarningsCountLogger _warningsCountLogger; + private const int CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS = 10000; + private const int CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS = 60000; + + public MigrateRepoCommandHandler( + OctoLogger log, + GithubApi githubApi, + GitlabApi gitlabApi, + EnvironmentVariableProvider environmentVariableProvider, + AzureApi azureApi, + AwsApi awsApi, + HttpDownloadService httpDownloadService, + FileSystemProvider fileSystemProvider, + WarningsCountLogger warningsCountLogger) + { + _log = log; + _githubApi = githubApi; + _gitlabApi = gitlabApi; + _azureApi = azureApi; + _awsApi = awsApi; + _httpDownloadService = httpDownloadService; + _environmentVariableProvider = environmentVariableProvider; + _fileSystemProvider = fileSystemProvider; + _warningsCountLogger = warningsCountLogger; + } + + public async Task Handle(MigrateRepoCommandArgs args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + ValidateOptions(args); + + await _gitlabApi.LogServerVersion(); + + var migrationSourceId = ""; + + if (args.ShouldImportArchive()) + { + var targetRepoExists = await _githubApi.DoesRepoExist(args.GithubOrg, args.GithubRepo); + + if (targetRepoExists) + { + throw new OctoshiftCliException($"A repository called {args.GithubOrg}/{args.GithubRepo} already exists"); + } + + migrationSourceId = await CreateMigrationSource(args); + } + + if (args.ShouldGenerateArchive()) + { + await GenerateArchive(args); + + _log.LogInformation($"Downloading GitLab archive..."); + + args.ArchivePath ??= _fileSystemProvider.GetTempFileName(); + await _gitlabApi.DownloadExportArchive(args.GitlabGroup, args.GitlabProject, args.ArchivePath); + + _log.LogInformation(args.KeepArchive ? $"Archive downloaded to \"{args.ArchivePath}\"" : "Archive download complete"); + } + + if (args.ShouldUploadArchive()) + { + _log.LogInformation($"Archive path: {args.ArchivePath}"); + + try + { + if (args.UseGithubStorage) + { + args.ArchiveUrl = await UploadArchiveToGithub(args.GithubOrg, args.ArchivePath); + } +#pragma warning disable IDE0045 + else if (args.AwsBucketName.HasValue()) +#pragma warning restore IDE0045 + { + args.ArchiveUrl = await UploadArchiveToAws(args.AwsBucketName, args.ArchivePath); + } + else + { + args.ArchiveUrl = await UploadArchiveToAzure(args.ArchivePath); + } + + } + finally + { + if (!args.KeepArchive) + { + DeleteArchive(args.ArchivePath); + } + } + } + + if (args.ShouldImportArchive()) + { + await ImportArchive(args, migrationSourceId, args.ArchiveUrl); + } + } + + private void DeleteArchive(string path) + { + try + { + _fileSystemProvider.DeleteIfExists(path); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _log.LogWarning($"Couldn't delete the downloaded archive. Error message: \"{ex.Message}\""); + _log.LogVerbose(ex.ToString()); + } + } + + private async Task GenerateArchive(MigrateRepoCommandArgs args) + { + await _gitlabApi.StartExport(args.GitlabGroup, args.GitlabProject); + + _log.LogInformation($"Export started."); + + var (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + + while (ExportState.IsInProgress(exportState)) + { + _log.LogInformation($"Export status: {exportState}."); + await Task.Delay(CHECK_EXPORT_STATUS_DELAY_IN_MILLISECONDS); + (exportState, archiveUrl) = await _gitlabApi.GetExport(args.GitlabGroup, args.GitlabProject); + } + + if (ExportState.IsError(exportState)) + { + throw new OctoshiftCliException($"GitLab archive export failed!"); + } + + _log.LogInformation($"Archive export completed."); + + return archiveUrl; + } + + private async Task UploadArchiveToAzure(string archivePath) + { + _log.LogInformation("Uploading Archive to Azure..."); + +#pragma warning disable IDE0063 + await using (var archiveData = _fileSystemProvider.OpenRead(archivePath)) +#pragma warning restore IDE0063 + { + var archiveName = GenerateArchiveName(); + var archiveBlobUrl = await _azureApi.UploadToBlob(archiveName, archiveData); + return archiveBlobUrl.ToString(); + } + } + + private string GenerateArchiveName() => $"{Guid.NewGuid()}.tar"; + + private async Task UploadArchiveToAws(string bucketName, string archivePath) + { + _log.LogInformation("Uploading Archive to AWS..."); + + var keyName = GenerateArchiveName(); + var archiveBlobUrl = await _awsApi.UploadToBucket(bucketName, archivePath, keyName); + + return archiveBlobUrl; + } + + private async Task UploadArchiveToGithub(string org, string archivePath) + { + await using var archiveData = _fileSystemProvider.OpenRead(archivePath); + var githubOrgDatabaseId = await _githubApi.GetOrganizationDatabaseId(org); + + _log.LogInformation("Uploading archive to GitHub Storage"); + var keyName = GenerateArchiveName(); + var authenticatedGitArchiveUri = await _githubApi.UploadArchiveToGithubStorage(githubOrgDatabaseId, keyName, archiveData); + + return authenticatedGitArchiveUri; + } + + private async Task CreateMigrationSource(MigrateRepoCommandArgs args) + { + _log.LogInformation("Creating Migration Source..."); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + try + { + return await _githubApi.CreateGitlabMigrationSource(githubOrgId); + } + catch (OctoshiftCliException ex) when (ex.Message.Contains("not have the correct permissions to execute")) + { + var insufficientPermissionsMessage = InsufficientPermissionsMessageGenerator.Generate(args.GithubOrg); + var message = $"{ex.Message}{insufficientPermissionsMessage}"; + throw new OctoshiftCliException(message, ex); + } + } + + private async Task ImportArchive(MigrateRepoCommandArgs args, string migrationSourceId, string archiveUrl = null) + { + _log.LogInformation("Importing Archive..."); + + archiveUrl ??= args.ArchiveUrl; + + var gitlabRepoUrl = GetGitlabProjectUrl(args); + + args.GithubPat ??= _environmentVariableProvider.TargetGithubPersonalAccessToken(); + var githubOrgId = await _githubApi.GetOrganizationId(args.GithubOrg); + + string migrationId; + + try + { + migrationId = await _githubApi.StartGitlabMigration(migrationSourceId, gitlabRepoUrl, githubOrgId, args.GithubRepo, args.GithubPat, archiveUrl, args.TargetRepoVisibility); + } + catch (OctoshiftCliException ex) when (ex.Message == $"A repository called {args.GithubOrg}/{args.GithubRepo} already exists") + { + _log.LogWarning($"The Org '{args.GithubOrg}' already contains a repository with the name '{args.GithubRepo}'. No operation will be performed"); + return; + } + + if (args.QueueOnly) + { + _log.LogInformation($"A repository migration (ID: {migrationId}) was successfully queued."); + return; + } + + var (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + + while (RepositoryMigrationStatus.IsPending(migrationState)) + { + _log.LogInformation($"Migration in progress (ID: {migrationId}). State: {migrationState}. Waiting 60 seconds..."); + await Task.Delay(CHECK_MIGRATION_STATUS_DELAY_IN_MILLISECONDS); + (migrationState, _, warningsCount, failureReason, migrationLogUrl) = await _githubApi.GetMigration(migrationId); + } + + var migrationLogAvailableMessage = $"Migration log available at {migrationLogUrl} or by running `gh {CliContext.RootCommand} download-logs --github-org {args.GithubOrg} --github-repo {args.GithubRepo}`"; + + if (RepositoryMigrationStatus.IsFailed(migrationState)) + { + _log.LogError($"Migration Failed. Migration ID: {migrationId}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + throw new OctoshiftCliException(failureReason); + } + + _log.LogSuccess($"Migration completed (ID: {migrationId})! State: {migrationState}"); + _warningsCountLogger.LogWarningsCount(warningsCount); + _log.LogInformation(migrationLogAvailableMessage); + } + + private string GetAwsAccessKey(MigrateRepoCommandArgs args) => args.AwsAccessKey.HasValue() ? args.AwsAccessKey : _environmentVariableProvider.AwsAccessKeyId(false); + + private string GetAwsSecretKey(MigrateRepoCommandArgs args) => args.AwsSecretKey.HasValue() ? args.AwsSecretKey : _environmentVariableProvider.AwsSecretAccessKey(false); + + private string GetAwsRegion(MigrateRepoCommandArgs args) => args.AwsRegion.HasValue() ? args.AwsRegion : _environmentVariableProvider.AwsRegion(false); + + private string GetAzureStorageConnectionString(MigrateRepoCommandArgs args) => args.AzureStorageConnectionString.HasValue() + ? args.AzureStorageConnectionString + : _environmentVariableProvider.AzureStorageConnectionString(false); + + private string GetGitlabPat(MigrateRepoCommandArgs args) => args.GitlabPat.HasValue() ? args.GitlabPat : _environmentVariableProvider.GitlabPat(false); + + private string GetGitlabProjectUrl(MigrateRepoCommandArgs args) + { + return args.GitlabServerUrl.HasValue() && args.GitlabGroup.HasValue() && args.GitlabProject.HasValue() + ? $"{args.GitlabServerUrl.TrimEnd('/')}/{args.GitlabGroup}/{args.GitlabProject}" + : "https://not-used"; + } + + private void ValidateOptions(MigrateRepoCommandArgs args) + { + if (args.ShouldGenerateArchive()) + { + if (GetGitlabPat(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("GitLab PAT must be either set as GITLAB_PAT environment variable or passed as --gitlab-pat."); + } + } + + // Validate --archive-path if provided as an input (i.e. not generating a new archive) + if (!args.ShouldGenerateArchive() && args.ArchivePath.HasValue() && !_fileSystemProvider.FileExists(args.ArchivePath)) + { + throw new OctoshiftCliException($"The archive file provided with --archive-path does not exist or is not accessible: {args.ArchivePath}"); + } + + if (args.ShouldUploadArchive()) + { + ValidateUploadOptions(args); + } + } + + private void ValidateUploadOptions(MigrateRepoCommandArgs args) + { + var shouldUseAzureStorage = GetAzureStorageConnectionString(args).HasValue(); + var shouldUseAwsS3 = args.AwsBucketName.HasValue(); + if (!shouldUseAzureStorage && !shouldUseAwsS3 && !args.UseGithubStorage) + { + throw new OctoshiftCliException( + "Either Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) or " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) or " + + "GitHub Storage Option (--use-github-storage) " + + "must be provided."); + } + + if (shouldUseAzureStorage && shouldUseAwsS3) + { + throw new OctoshiftCliException( + "Azure storage connection (--azure-storage-connection-string or AZURE_STORAGE_CONNECTION_STRING env. variable) and " + + "AWS S3 connection (--aws-bucket-name, --aws-access-key (or AWS_ACCESS_KEY_ID env. variable), --aws-secret-key (or AWS_SECRET_ACCESS_KEY env.variable)) cannot be " + + "specified together."); + } + + if (shouldUseAwsS3) + { + if (!GetAwsAccessKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-access-key or AWS_ACCESS_KEY_ID environment variable must be set."); + } + + if (!GetAwsSecretKey(args).HasValue()) + { + throw new OctoshiftCliException("Either --aws-secret-key or AWS_SECRET_ACCESS_KEY environment variable must be set."); + } + + if (GetAwsRegion(args).IsNullOrWhiteSpace()) + { + throw new OctoshiftCliException("Either --aws-region or AWS_REGION environment variable must be set."); + } + } + } +} diff --git a/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs new file mode 100644 index 000000000..587ed556c --- /dev/null +++ b/src/gl2gh/Commands/ReclaimMannequin/ReclaimMannequinCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.ReclaimMannequin; + +namespace OctoshiftCLI.GitlabToGithub.Commands.ReclaimMannequin; + +public sealed class ReclaimMannequinCommand : ReclaimMannequinCommandBase +{ + public ReclaimMannequinCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs new file mode 100644 index 000000000..b94e13975 --- /dev/null +++ b/src/gl2gh/Commands/RevokeMigratorRole/RevokeMigratorRoleCommand.cs @@ -0,0 +1,8 @@ +using OctoshiftCLI.Commands.RevokeMigratorRole; + +namespace OctoshiftCLI.GitlabToGithub.Commands.RevokeMigratorRole; + +public sealed class RevokeMigratorRoleCommand : RevokeMigratorRoleCommandBase +{ + public RevokeMigratorRoleCommand() => AddOptions(); +} diff --git a/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs new file mode 100644 index 000000000..0a03bd1a9 --- /dev/null +++ b/src/gl2gh/Commands/WaitForMigration/WaitForMigrationCommand.cs @@ -0,0 +1,7 @@ +using OctoshiftCLI.Commands.WaitForMigration; +namespace OctoshiftCLI.GitlabToGithub.Commands.WaitForMigration; + +public sealed class WaitForMigrationCommand : WaitForMigrationCommandBase +{ + public WaitForMigrationCommand() => AddOptions(); +} diff --git a/src/gl2gh/ExportState.cs b/src/gl2gh/ExportState.cs new file mode 100644 index 000000000..c403e3b56 --- /dev/null +++ b/src/gl2gh/ExportState.cs @@ -0,0 +1,11 @@ +namespace OctoshiftCLI.GitlabToGithub; + +public static class ExportState +{ + public const string FINISHED = "finished"; + public const string FAILED = "failed"; + + public static bool IsInProgress(string state) => state is not FINISHED && !IsError(state); + + public static bool IsError(string state) => state is FAILED; +} diff --git a/src/gl2gh/Factories/GitlabApiFactory.cs b/src/gl2gh/Factories/GitlabApiFactory.cs new file mode 100644 index 000000000..b5c4ad3af --- /dev/null +++ b/src/gl2gh/Factories/GitlabApiFactory.cs @@ -0,0 +1,42 @@ +using System.Net.Http; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabApiFactory +{ + private readonly OctoLogger _octoLogger; + private readonly IHttpClientFactory _clientFactory; + private readonly EnvironmentVariableProvider _environmentVariableProvider; + private readonly IVersionProvider _versionProvider; + private readonly RetryPolicy _retryPolicy; + private readonly FileSystemProvider _fileSystemProvider; + + public GitlabApiFactory( + OctoLogger octoLogger, + IHttpClientFactory clientFactory, + EnvironmentVariableProvider environmentVariableProvider, + IVersionProvider versionProvider, + RetryPolicy retryPolicy, + FileSystemProvider fileSystemProvider) + { + _octoLogger = octoLogger; + _clientFactory = clientFactory; + _environmentVariableProvider = environmentVariableProvider; + _versionProvider = versionProvider; + _retryPolicy = retryPolicy; + _fileSystemProvider = fileSystemProvider; + } + + public virtual GitlabApi Create(string gitlabServerUrl, string gitlabPat, bool noSsl = false) + { + gitlabPat ??= _environmentVariableProvider.GitlabPat(); + + var httpClient = noSsl ? _clientFactory.CreateClient("NoSSL") : _clientFactory.CreateClient("Default"); + + var clientRetryPolicy = (_retryPolicy ?? new RetryPolicy(_octoLogger)).WithServiceName("GitLab"); + var gitlabClient = new GitlabClient(_octoLogger, httpClient, _versionProvider, clientRetryPolicy, gitlabPat, _fileSystemProvider); + return new GitlabApi(gitlabClient, gitlabServerUrl, _octoLogger); + } +} diff --git a/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs new file mode 100644 index 000000000..131aef62b --- /dev/null +++ b/src/gl2gh/Factories/GitlabInspectorServiceFactory.cs @@ -0,0 +1,18 @@ +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub.Factories; + +public class GitlabInspectorServiceFactory +{ + private readonly OctoLogger _octoLogger; + private GitlabInspectorService _instance; + + public GitlabInspectorServiceFactory(OctoLogger octoLogger) => _octoLogger = octoLogger; + + public virtual GitlabInspectorService Create(GitlabApi gitlabApi) + { + _instance ??= new(_octoLogger, gitlabApi); + + return _instance; + } +} diff --git a/src/gl2gh/Program.cs b/src/gl2gh/Program.cs new file mode 100644 index 000000000..dd9e85cb8 --- /dev/null +++ b/src/gl2gh/Program.cs @@ -0,0 +1,150 @@ +using System; +using System.CommandLine; +using System.CommandLine.Builder; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; + +using OctoshiftCLI.GitlabToGithub.Factories; +using OctoshiftCLI.Contracts; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Factories; +using OctoshiftCLI.Services; + +[assembly: InternalsVisibleTo("OctoshiftCLI.Tests")] +namespace OctoshiftCLI.GitlabToGithub +{ + public static class Program + { + private static readonly OctoLogger Logger = new(); + + [SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "If the version check fails for any reason, we want the CLI to carry on with the current command")] + public static async Task Main(string[] args) + { + Logger.LogDebug("Execution Started"); + + var serviceCollection = new ServiceCollection(); + serviceCollection + .AddSingleton(Logger) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(sp => sp.GetRequiredService()) + .AddSingleton() + .AddHttpClient("NoSSL", noSsl: true) + .AddHttpClient("Default"); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + var rootCommand = new RootCommand("Automate end-to-end GitLab to GitHub migrations.") + .AddCommands(serviceProvider); + + var commandLineBuilder = new CommandLineBuilder(rootCommand); + var parser = commandLineBuilder + .UseDefaults() + .UseExceptionHandler((ex, _) => + { + Logger.LogError(ex); + Environment.ExitCode = 1; + }, 1) + .Build(); + + SetContext(new InvocationContext(parser.Parse(args))); + + try + { + await GithubStatusCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not check GitHub availability from githubstatus.com. See https://www.githubstatus.com for details."); + Logger.LogVerbose(ex.ToString()); + } + + try + { + await LatestVersionCheck(serviceProvider); + } + catch (Exception ex) + { + Logger.LogWarning("Could not retrieve latest gl2gh extension version from github.com, please ensure you are using the latest version by running: gh extension upgrade gl2gh"); + Logger.LogVerbose(ex.ToString()); + } + + await parser.InvokeAsync(args); + } + + private static void SetContext(InvocationContext context) + { + CliContext.RootCommand = "gl2gh"; + CliContext.ExecutingCommand = context.ParseResult.CommandResult.Command.Name; + } + + private static async Task GithubStatusCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipStatusCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped GitHub status check due to GEI_SKIP_STATUS_CHECK environment variable"); + return; + } + + var githubStatusApi = sp.GetRequiredService(); + + if (await githubStatusApi.GetUnresolvedIncidentsCount() > 0) + { + Logger.LogWarning("GitHub is currently experiencing availability issues. See https://www.githubstatus.com for details."); + } + } + + private static async Task LatestVersionCheck(ServiceProvider sp) + { + var envProvider = sp.GetRequiredService(); + + if (envProvider.SkipVersionCheck()?.ToUpperInvariant() is "TRUE" or "1") + { + Logger.LogInformation("Skipped latest version check due to GEI_SKIP_VERSION_CHECK environment variable"); + return; + } + + var versionChecker = sp.GetRequiredService(); + + if (await versionChecker.IsLatest()) + { + Logger.LogInformation($"You are running an up-to-date version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]"); + } + else + { + Logger.LogWarning($"You are running an old version of the gl2gh extension [v{versionChecker.GetCurrentVersion()}]. The latest version is v{await versionChecker.GetLatestVersion()}."); + Logger.LogWarning($"Please update by running: gh extension upgrade gl2gh"); + } + } + + private static IServiceCollection AddHttpClient(this IServiceCollection serviceCollection, string name, bool noSsl = false) => serviceCollection + .AddHttpClient(name, _ => { }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + ServerCertificateCustomValidationCallback = noSsl ? delegate { return true; } : null + }) + .Services; + } +} diff --git a/src/gl2gh/Services/GitlabInspectorService.cs b/src/gl2gh/Services/GitlabInspectorService.cs new file mode 100644 index 000000000..b7100510f --- /dev/null +++ b/src/gl2gh/Services/GitlabInspectorService.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Octoshift.Models; +using OctoshiftCLI.Extensions; +using OctoshiftCLI.Services; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GitlabInspectorService + { + private readonly OctoLogger _log; + private readonly GitlabApi _gitlabApi; + + private IList<(string, string)> _groups; + private readonly IDictionary> _repos = new Dictionary>(); + private readonly IDictionary> _mrCounts = new Dictionary>(); + + public GitlabInspectorService(OctoLogger log, GitlabApi gitlabApi) + { + _log = log; + _gitlabApi = gitlabApi; + } + + public virtual async Task> GetGroups() + { + if (_groups is null) + { + _log.LogInformation($"Retrieving list of all Groups the user has access to..."); + _groups = (await _gitlabApi.GetGroups()) + .Select(group => (group.Path, group.Name)) + .ToList(); + } + + return _groups; + } + + public virtual async Task<(string Key, string Name)> GetGroup(string groupPath) + { + _log.LogInformation($"Retrieving Group..."); + var (_, Key, Name) = await _gitlabApi.GetGroup(groupPath); + + return (Key, Name); + } + + public virtual async Task> GetProjects(string groupPath) + { + if (!_repos.TryGetValue(groupPath, out var repos)) + { + repos = (await _gitlabApi.GetProjects(groupPath)) + .Select(repo => new GitlabProject() { Name = repo.Name, Path = repo.Path, Archived = repo.Archived }) + .ToList(); + _repos.Add(groupPath, repos); + } + + return repos; + } + + public virtual async Task GetProjectCount(string[] groups) + { + return await groups.Sum(async key => await GetProjectCount(key)); + } + + public virtual async Task GetProjectCount() + { + var groups = await GetGroups(); + return await groups.Sum(async group => await GetProjectCount(group.Path)); + } + + public virtual async Task GetProjectCount(string groupPath) + { + return (await GetProjects(groupPath)).Count(); + } + + public virtual async Task GetMergeRequestCount(string groupPath) + { + var repos = await GetProjects(groupPath); + return await repos.Sum(async repo => await GetProjectMergeRequestCount(groupPath, repo.Path)); + } + + public virtual async Task GetProjectMergeRequestCount(string groupPath, string repo) + { + if (!_mrCounts.ContainsKey(groupPath)) + { + _mrCounts.Add(groupPath, new Dictionary()); + } + + if (!_mrCounts[groupPath].TryGetValue(repo, out var mrCount)) + { + mrCount = await _gitlabApi.GetMergeRequestCount(groupPath, repo); + _mrCounts[groupPath][repo] = mrCount; + } + + return mrCount; + } + } +} diff --git a/src/gl2gh/Services/GroupsCsvGeneratorService.cs b/src/gl2gh/Services/GroupsCsvGeneratorService.cs new file mode 100644 index 000000000..0a473dac2 --- /dev/null +++ b/src/gl2gh/Services/GroupsCsvGeneratorService.cs @@ -0,0 +1,47 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class GroupsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public GroupsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + { + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("project-key,project-name,url,repo-count"); + result.AppendLine(!minimal ? ",mr-count" : null); + + var projects = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; + + foreach (var (Key, Name) in projects) + { + var url = $"{gitlabServerUrl.TrimEnd('/')}/projects/{Uri.EscapeDataString(Key)}"; + var repoCount = await inspector.GetProjectCount(Key); + var mrCount = !minimal ? await inspector.GetMergeRequestCount(Key) : 0; + + var projectName = Name.Replace(",", Uri.EscapeDataString(",")); + + result.Append($"\"{Key}\",\"{projectName}\",\"{url}\",{repoCount}"); + result.AppendLine(!minimal ? $",{mrCount}" : null); + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/Services/ProjectsCsvGeneratorService.cs b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs new file mode 100644 index 000000000..4e4f3a93b --- /dev/null +++ b/src/gl2gh/Services/ProjectsCsvGeneratorService.cs @@ -0,0 +1,60 @@ +using System; +using System.Text; +using System.Threading.Tasks; +using OctoshiftCLI.GitlabToGithub.Factories; + +namespace OctoshiftCLI.GitlabToGithub +{ + public class ProjectsCsvGeneratorService + { + private readonly GitlabInspectorServiceFactory _gitlabInspectorServiceFactory; + private readonly GitlabApiFactory _gitlabApiFactory; + + public ProjectsCsvGeneratorService(GitlabInspectorServiceFactory gitlabInspectorServiceFactory, GitlabApiFactory gitlabApiFactory) + { + _gitlabInspectorServiceFactory = gitlabInspectorServiceFactory; + _gitlabApiFactory = gitlabApiFactory; + } + + public virtual async Task Generate(string gitlabServerUrl, string gitlabPat, bool noSslVerify, string gitlabGroup = "", bool minimal = false) + { + gitlabServerUrl = gitlabServerUrl ?? throw new ArgumentNullException(nameof(gitlabServerUrl)); + + var gitlabApi = _gitlabApiFactory.Create(gitlabServerUrl, gitlabPat, noSslVerify); + var inspector = _gitlabInspectorServiceFactory.Create(gitlabApi); + var result = new StringBuilder(); + + result.Append("group-path,group-name,project,url,last-commit-date,repo-size-in-bytes,attachments-size-in-bytes,is-archived"); + result.AppendLine(!minimal ? ",mr-count" : null); + + var groups = string.IsNullOrWhiteSpace(gitlabGroup) ? await inspector.GetGroups() : new[] { await inspector.GetGroup(gitlabGroup) }; + + foreach (var (groupPath, groupName) in groups) + { + foreach (var project in await inspector.GetProjects(groupPath)) + { + var url = $"{gitlabServerUrl.TrimEnd('/')}/{groupPath}/{project.Path}"; + var lastCommitDate = await gitlabApi.GetRepositoryLatestCommitDate(groupPath, project.Path); + var (repoSize, attachmentsSize) = await gitlabApi.GetRepositoryAndAttachmentsSize(groupPath, project.Path); + var mrCount = !minimal ? await inspector.GetProjectMergeRequestCount(groupPath, project.Path) : 0; + + var group = groupName.Replace(",", Uri.EscapeDataString(",")); + var projectName = project.Name.Replace(",", Uri.EscapeDataString(",")); + + if (lastCommitDate == null) + { + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",,\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); + } + else + { + result.Append($"\"{groupPath}\",\"{group}\",\"{projectName}\",\"{url}\",\"{lastCommitDate:yyyy-MM-dd hh:mm tt}\",\"{repoSize:D}\",\"{attachmentsSize:D}\",\"{project.Archived}\""); + } + + result.AppendLine(!minimal ? $",{mrCount}" : null); + } + } + + return result.ToString(); + } + } +} diff --git a/src/gl2gh/gl2gh.csproj b/src/gl2gh/gl2gh.csproj new file mode 100644 index 000000000..2088f3769 --- /dev/null +++ b/src/gl2gh/gl2gh.csproj @@ -0,0 +1,23 @@ + + + + Exe + net8.0 + gl2gh + 12 + OctoshiftCLI.GitlabToGithub + + + + + + + + + + + + + + +