diff --git a/GitDensity/App.config b/GitDensity/App.config index 12259fc..be33137 100644 --- a/GitDensity/App.config +++ b/GitDensity/App.config @@ -47,7 +47,7 @@ - + @@ -79,7 +79,7 @@ - + @@ -109,6 +109,10 @@ + + + + diff --git a/GitDensity/GitDensity.csproj b/GitDensity/GitDensity.csproj index 2805e15..88ca784 100644 --- a/GitDensity/GitDensity.csproj +++ b/GitDensity/GitDensity.csproj @@ -85,8 +85,8 @@ ..\packages\Microsoft.Extensions.Logging.6.0.0\lib\net461\Microsoft.Extensions.Logging.dll - - ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.1\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll + + ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.2\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll ..\packages\Microsoft.Extensions.Logging.Configuration.6.0.0\lib\net461\Microsoft.Extensions.Logging.Configuration.dll @@ -110,7 +110,7 @@ ..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll - ..\packages\NHibernate.5.3.12\lib\net461\NHibernate.dll + ..\packages\NHibernate.5.3.13\lib\net461\NHibernate.dll ..\packages\Remotion.Linq.2.2.0\lib\net45\Remotion.Linq.dll @@ -118,8 +118,8 @@ ..\packages\Remotion.Linq.EagerFetching.2.2.0\lib\net45\Remotion.Linq.EagerFetching.dll - - ..\packages\SSH.NET.2020.0.1\lib\net40\Renci.SshNet.dll + + ..\packages\SSH.NET.2020.0.2\lib\net40\Renci.SshNet.dll @@ -139,14 +139,14 @@ True - - ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\lib\net46\System.Data.SQLite.dll + + ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.116.0\lib\net46\System.Data.SQLite.dll - - ..\packages\System.Data.SQLite.EF6.1.0.115.5\lib\net46\System.Data.SQLite.EF6.dll + + ..\packages\System.Data.SQLite.EF6.1.0.116.0\lib\net46\System.Data.SQLite.EF6.dll - - ..\packages\System.Data.SQLite.Linq.1.0.115.5\lib\net46\System.Data.SQLite.Linq.dll + + ..\packages\System.Data.SQLite.Linq.1.0.116.0\lib\net46\System.Data.SQLite.Linq.dll ..\packages\System.Diagnostics.DiagnosticSource.6.0.0\lib\net461\System.Diagnostics.DiagnosticSource.dll @@ -161,8 +161,8 @@ ..\packages\System.IO.FileSystem.Primitives.4.3.0\lib\net46\System.IO.FileSystem.Primitives.dll - - ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll ..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll @@ -197,8 +197,8 @@ ..\packages\System.Text.Encodings.Web.6.0.0\lib\net461\System.Text.Encodings.Web.dll - - ..\packages\System.Text.Json.6.0.0\lib\net461\System.Text.Json.dll + + ..\packages\System.Text.Json.6.0.6\lib\net461\System.Text.Json.dll ..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll @@ -250,13 +250,13 @@ - - - + + + - - - + + + \ No newline at end of file diff --git a/GitDensity/Program.cs b/GitDensity/Program.cs index 68d8110..399b520 100644 --- a/GitDensity/Program.cs +++ b/GitDensity/Program.cs @@ -221,7 +221,8 @@ static void Main(string[] args) repoTempPath, useRepoName: useRepoName, pullIfAlreadyExists: true)) { logger.LogInformation($"Repository is located in {repo.Info.WorkingDirectory}"); - var span = new GitCommitSpan(repo, options.Since, options.Until); + var span = new GitCommitSpan(repo, options.Since, options.Until, + sinceUseDate: options.SinceUseDate, untilUseDate: options.UntilUseDate); // Instantiate the Density analysis with the selected programming // languages' file extensions and other options from the command line. @@ -301,9 +302,6 @@ internal class CommandLineOptions [Option('r', "repo-path", Required = true, HelpText = "Absolute path or HTTP(S) URL to a git-repository. If a URL is provided, the repository will be cloned to a temporary folder first, using its defined default branch. Also allows passing in an Internal-ID of a project from the database.")] public String RepoPath { get; set; } - [Option('c', "config-file", Required = false, HelpText = "Optional. Absolute path to a valid configuration.json. If not given, uses the configuration.json that is to be found in the same folder as " + nameof(GitDensity) + ".exe.")] - public String ConfigFile { get; set; } - /// /// To obtains the actual s, use the /// property . @@ -311,6 +309,9 @@ internal class CommandLineOptions [OptionList('p', "prog-langs", ',', Required = true, HelpText = "A comma-separated list of programming languages to examine in the given repository. Other files will be ignored.")] public IList LanguagesRaw { get; set; } + [Option('c', "config-file", Required = false, HelpText = "Optional. Absolute path to a valid configuration.json. If not given, uses the configuration.json that is to be found in the same folder as " + nameof(GitDensity) + ".exe.")] + public String ConfigFile { get; set; } + [Option('i', "skip-initial-commit", Required = false, DefaultValue = false, HelpText = "If present, does not analyze the pair that consists of the 2nd and the initial commit to a repository.")] public Boolean SkipInitialCommit { get; set; } @@ -327,9 +328,17 @@ internal class CommandLineOptions [Option('s', "since", Required = false, HelpText = "Optional. Analyze data since a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Since { get; set; } + [Option("since-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using a since-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate SinceUseDate { get; set; } + [Option('u', "until", Required = false, HelpText = "Optional. Analyze data until (inclusive) a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Until { get; set; } + [Option("until-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using an until-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate UntilUseDate { get; set; } + [Option('e', "exec-policy", Required = false, DefaultValue = ExecutionPolicy.Parallel, HelpText = "Optional. Set the execution policy for the analysis. Allowed values are " + nameof(ExecutionPolicy.Parallel) + " and " + nameof(ExecutionPolicy.Linear) + ". The former is faster while the latter uses only minimal resources.")] [JsonConverter(typeof(StringEnumConverter))] public ExecutionPolicy ExecutionPolicy { get; set; } diff --git a/GitDensity/packages.config b/GitDensity/packages.config index aa02294..c6a480e 100644 --- a/GitDensity/packages.config +++ b/GitDensity/packages.config @@ -15,31 +15,31 @@ - + - + - + - - + + - - - - + + + + @@ -53,7 +53,7 @@ - + @@ -78,7 +78,7 @@ - + diff --git a/GitDensityTests/app.config b/GitDensityTests/app.config index 766f877..400f96e 100644 --- a/GitDensityTests/app.config +++ b/GitDensityTests/app.config @@ -20,7 +20,7 @@ - + @@ -52,7 +52,7 @@ - + @@ -82,6 +82,10 @@ + + + + \ No newline at end of file diff --git a/GitHours/App.config b/GitHours/App.config index 5693db6..0d93bf6 100644 --- a/GitHours/App.config +++ b/GitHours/App.config @@ -44,7 +44,7 @@ - + @@ -56,7 +56,7 @@ - + diff --git a/GitHours/GitHours.csproj b/GitHours/GitHours.csproj index c80fb2b..ebf29e6 100644 --- a/GitHours/GitHours.csproj +++ b/GitHours/GitHours.csproj @@ -66,8 +66,8 @@ ..\packages\Microsoft.Extensions.Logging.6.0.0\lib\net461\Microsoft.Extensions.Logging.dll - - ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.1\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll + + ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.2\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll ..\packages\Microsoft.Extensions.Options.6.0.0\lib\net461\Microsoft.Extensions.Options.dll @@ -96,8 +96,8 @@ ..\packages\System.Diagnostics.DiagnosticSource.6.0.0\lib\net461\System.Diagnostics.DiagnosticSource.dll - - ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll @@ -147,7 +147,7 @@ This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + \ No newline at end of file diff --git a/GitHours/Program.cs b/GitHours/Program.cs index 884498e..fb00600 100644 --- a/GitHours/Program.cs +++ b/GitHours/Program.cs @@ -18,6 +18,7 @@ using LibGit2Sharp; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; using System; using System.Globalization; using System.IO; @@ -132,7 +133,8 @@ static void Main(string[] args) try { using (repository) - using (var span = new GitCommitSpan(repository, options.Since, options.Until)) + using (var span = new GitCommitSpan(repository, options.Since, options.Until, + sinceUseDate: options.SinceUseDate, untilUseDate: options.UntilUseDate)) { var gitHours = new Hours.GitHours(span, options.MaxCommitDiff, options.FirstCommitAdd); @@ -193,9 +195,17 @@ internal class CommandLineOptions [Option('s', "since", Required = false, HelpText = "Optional. Analyze data since a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Since { get; set; } + [Option("since-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using a since-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate SinceUseDate { get; set; } + [Option('u', "until", Required = false, HelpText = "Optional. Analyze data until (inclusive) a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Until { get; set; } + [Option("until-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using an until-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate UntilUseDate { get; set; } + [Option('t', "temp-dir", Required = false, HelpText = "Optional. A fully qualified path to a custom temporary directory. If not specified, will use the system's default.")] public String TempDirectory { get; set; } diff --git a/GitHours/packages.config b/GitHours/packages.config index 78faba0..8765589 100644 --- a/GitHours/packages.config +++ b/GitHours/packages.config @@ -10,7 +10,7 @@ - + @@ -24,7 +24,7 @@ - + diff --git a/GitHoursTests/app.config b/GitHoursTests/app.config index 89886e4..d624950 100644 --- a/GitHoursTests/app.config +++ b/GitHoursTests/app.config @@ -36,7 +36,7 @@ - + @@ -48,7 +48,7 @@ - + diff --git a/GitMetrics/App.config b/GitMetrics/App.config index ff0df40..dc22d26 100644 --- a/GitMetrics/App.config +++ b/GitMetrics/App.config @@ -32,7 +32,7 @@ - + @@ -52,7 +52,7 @@ - + diff --git a/GitMetrics/GitMetrics.csproj b/GitMetrics/GitMetrics.csproj index 9cee9b1..907c8af 100644 --- a/GitMetrics/GitMetrics.csproj +++ b/GitMetrics/GitMetrics.csproj @@ -75,8 +75,8 @@ ..\packages\Microsoft.Extensions.Logging.6.0.0\lib\net461\Microsoft.Extensions.Logging.dll - - ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.1\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll + + ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.2\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll ..\packages\Microsoft.Extensions.Options.6.0.0\lib\net461\Microsoft.Extensions.Options.dll @@ -88,7 +88,7 @@ ..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll - ..\packages\NHibernate.5.3.12\lib\net461\NHibernate.dll + ..\packages\NHibernate.5.3.13\lib\net461\NHibernate.dll ..\packages\Remotion.Linq.2.2.0\lib\net45\Remotion.Linq.dll @@ -109,8 +109,8 @@ ..\packages\System.Diagnostics.DiagnosticSource.6.0.0\lib\net461\System.Diagnostics.DiagnosticSource.dll - - ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll @@ -162,7 +162,7 @@ This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + \ No newline at end of file diff --git a/GitMetrics/Program.cs b/GitMetrics/Program.cs index 2bc41dd..7bf5ee1 100644 --- a/GitMetrics/Program.cs +++ b/GitMetrics/Program.cs @@ -184,7 +184,8 @@ static void Main(string[] args) } using (repository) - using (var span = new GitCommitSpan(repository, options.Since, options.Until)) + using (var span = new GitCommitSpan(repository, options.Since, options.Until, + sinceUseDate: options.SinceUseDate, untilUseDate: options.UntilUseDate)) { var repoEntity = repository.AsEntity(span); var commitEntities = span.Select(commit => commit.AsEntity(repoEntity)).ToList(); @@ -259,9 +260,17 @@ internal class CommandLineOptions [Option('s', "since", Required = false, HelpText = "Optional. Analyze data since a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Since { get; set; } + [Option("since-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using a since-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate SinceUseDate { get; set; } + [Option('u', "until", Required = false, HelpText = "Optional. Analyze data until (inclusive) a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Until { get; set; } + [Option("until-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using an until-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate UntilUseDate { get; set; } + [Option('t', "temp-dir", Required = false, HelpText = "Optional. A fully qualified path to a custom temporary directory. If not specified, will use the system's default.")] public String TempDirectory { get; set; } diff --git a/GitMetrics/QualityAnalyzer/RepositoryAnalyzer.cs b/GitMetrics/QualityAnalyzer/RepositoryAnalyzer.cs index 121229c..07f7e25 100644 --- a/GitMetrics/QualityAnalyzer/RepositoryAnalyzer.cs +++ b/GitMetrics/QualityAnalyzer/RepositoryAnalyzer.cs @@ -19,9 +19,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; using Util; using Util.Data.Entities; @@ -112,39 +110,44 @@ public void Analyze() parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount / 2; } - Parallel.ForEach(this.CommitEntities, parallelOptions, commit => { - var copyRepo = this.Repository.BundleAndCloneTo( - Configuration.TempDirectory.FullName); - try - { - logger.LogDebug($"Checking out repository at commit {commit.BaseObject.ShaShort()}"); - Commands.Checkout(copyRepo, commit.BaseObject); - } - catch (Exception ex) - { - logger.LogError($"Cannot checkout commit {commit.BaseObject.ShaShort()}: {ex.Message}", ex); - - this.Results.Add(new MetricsAnalysisResult + using (var repoBundleCollection = this.Repository.CreateBundleCollection( + targetPath: Configuration.TempDirectory.FullName, + numInstances: (UInt16)parallelOptions.MaxDegreeOfParallelism, + deleteClonedReposAfterwards: this.DeleteClonedRepoAfterwards + )) { + Parallel.ForEach(this.CommitEntities, parallelOptions, commit => { + using (var loanedRepo = repoBundleCollection.Loan()) + // using and dispose will return the item to the collection. { - Commit = commit, - Metrics = new List(), - MetricTypes = new List(), - Repository = this.RepositoryEntity, - CommitMetricsStatus = CommitMetricsStatus.CheckoutError - }); - return; - } - - var analyzer = CreateAnalyzer( - this.Configuration, analyzerTypeName, analyzerTypeName.Contains(".")); - - this.Results.Add(analyzer.Analyze(copyRepo, this.RepositoryEntity, commit)); - - if (this.DeleteClonedRepoAfterwards) - { - new DirectoryInfo(copyRepo.Info.WorkingDirectory).TryDelete(); - } - }); + var copyRepo = loanedRepo.Item; + + try + { + logger.LogDebug($"Checking out repository at commit {commit.BaseObject.ShaShort()}"); + Commands.Checkout(copyRepo, commit.BaseObject); + } + catch (Exception ex) + { + logger.LogError($"Cannot checkout commit {commit.BaseObject.ShaShort()}: {ex.Message}", ex); + + this.Results.Add(new MetricsAnalysisResult + { + Commit = commit, + Metrics = new List(), + MetricTypes = new List(), + Repository = this.RepositoryEntity, + CommitMetricsStatus = CommitMetricsStatus.CheckoutError + }); + return; + } + + var analyzer = CreateAnalyzer( + this.Configuration, analyzerTypeName, analyzerTypeName.Contains(".")); + + this.Results.Add(analyzer.Analyze(copyRepo, this.RepositoryEntity, commit)); + } + }); + } } #region ISupportsExecPol diff --git a/GitMetrics/packages.config b/GitMetrics/packages.config index ed572ab..ed0938b 100644 --- a/GitMetrics/packages.config +++ b/GitMetrics/packages.config @@ -13,17 +13,17 @@ - + - + - + diff --git a/GitTools/Analysis/ExtendedAnalyzer/ExtendedAnalyzer.cs b/GitTools/Analysis/ExtendedAnalyzer/ExtendedAnalyzer.cs index 8bb9fd8..5a55d65 100644 --- a/GitTools/Analysis/ExtendedAnalyzer/ExtendedAnalyzer.cs +++ b/GitTools/Analysis/ExtendedAnalyzer/ExtendedAnalyzer.cs @@ -140,7 +140,7 @@ public override IEnumerable AnalyzeCommits() { // We do this to avoid thread-congestion while still achieving // a respectable CPU usage, as the loop-callback is IO-bound. - po.MaxDegreeOfParallelism = Environment.ProcessorCount * 64; + po.MaxDegreeOfParallelism = Environment.ProcessorCount * 4; this.Logger.LogDebug($"Using maximum degree of parallelism = {po.MaxDegreeOfParallelism} in {nameof(ExtendedAnalyzer)}."); } diff --git a/GitTools/Analysis/ExtendedAnalyzer/ExtendedCommitDetails.cs b/GitTools/Analysis/ExtendedAnalyzer/ExtendedCommitDetails.cs index 0642ef8..2a23bb3 100644 --- a/GitTools/Analysis/ExtendedAnalyzer/ExtendedCommitDetails.cs +++ b/GitTools/Analysis/ExtendedAnalyzer/ExtendedCommitDetails.cs @@ -40,7 +40,7 @@ public ExtendedCommitDetails(String repoPathOrUrl, Commit commit) /// Overridden to return the entire commit message, which is potentially multi-line. /// Line-breaks are substituted with single spaces. /// - [CsvColumn(FieldIndex = 8)] + [CsvColumn(FieldIndex = 9)] public override string Message => RegexNewLines.Replace(base.commit.Message, " ").Replace('"', ' ').Trim(); #endregion @@ -50,74 +50,74 @@ public override string Message /// For initial commits (without parent), this field has a negative value (i.e. -.1) /// to make it easier to distinguish from other commits' values. /// - [CsvColumn(FieldIndex = 7)] + [CsvColumn(FieldIndex = 10)] public double MinutesSincePreviousCommit { get; protected internal set; } = -.1; - [CsvColumn(FieldIndex = 11)] + [CsvColumn(FieldIndex = 13)] public String AuthorNominalLabel { get; protected internal set; } = String.Empty; - [CsvColumn(FieldIndex = 12)] + [CsvColumn(FieldIndex = 14)] public String CommitterNominalLabel { get; protected internal set; } = String.Empty; - [CsvColumn(FieldIndex = 17)] + [CsvColumn(FieldIndex = 19)] public UInt32 NumberOfFilesAdded { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 18)] + [CsvColumn(FieldIndex = 20)] public UInt32 NumberOfFilesAddedNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 19)] + [CsvColumn(FieldIndex = 21)] public UInt32 NumberOfLinesAddedByAddedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 20)] + [CsvColumn(FieldIndex = 22)] public UInt32 NumberOfLinesAddedByAddedFilesNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 21)] + [CsvColumn(FieldIndex = 23)] public UInt32 NumberOfFilesDeleted { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 22)] + [CsvColumn(FieldIndex = 24)] public UInt32 NumberOfFilesDeletedNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 23)] + [CsvColumn(FieldIndex = 25)] public UInt32 NumberOfLinesDeletedByDeletedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 24)] + [CsvColumn(FieldIndex = 26)] public UInt32 NumberOfLinesDeletedByDeletedFilesNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 25)] + [CsvColumn(FieldIndex = 27)] public UInt32 NumberOfFilesModified { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 26)] + [CsvColumn(FieldIndex = 28)] public UInt32 NumberOfFilesModifiedNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 27)] + [CsvColumn(FieldIndex = 29)] public UInt32 NumberOfFilesRenamed { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 28)] + [CsvColumn(FieldIndex = 30)] public UInt32 NumberOfFilesRenamedNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 29)] + [CsvColumn(FieldIndex = 31)] public UInt32 NumberOfLinesAddedByModifiedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 30)] + [CsvColumn(FieldIndex = 32)] public UInt32 NumberOfLinesAddedByModifiedFilesNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 31)] + [CsvColumn(FieldIndex = 33)] public UInt32 NumberOfLinesDeletedByModifiedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 32)] + [CsvColumn(FieldIndex = 34)] public UInt32 NumberOfLinesDeletedByModifiedFilesNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 33)] + [CsvColumn(FieldIndex = 35)] public UInt32 NumberOfLinesAddedByRenamedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 34)] + [CsvColumn(FieldIndex = 36)] public UInt32 NumberOfLinesAddedByRenamedFilesNet { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 35)] + [CsvColumn(FieldIndex = 37)] public UInt32 NumberOfLinesDeletedByRenamedFiles { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 36)] + [CsvColumn(FieldIndex = 38)] public UInt32 NumberOfLinesDeletedByRenamedFilesNet { get; protected internal set; } = 0u; #endregion @@ -126,7 +126,7 @@ public override string Message /// Returns the ratio between all lines added/removed gross and all lines /// added/removed net ([0,1]). /// - [CsvColumn(FieldIndex = 37)] + [CsvColumn(FieldIndex = 39)] public Double Density { get @@ -154,7 +154,7 @@ public Double Density /// the amount of files affected net (an affected file is one that had /// more than zero lines net added/deleted after). /// - [CsvColumn(FieldIndex = 38)] + [CsvColumn(FieldIndex = 40)] public Double AffectedFilesRatioNet { get diff --git a/GitTools/Analysis/SimpleAnalyzer/SimpleCommitDetails.cs b/GitTools/Analysis/SimpleAnalyzer/SimpleCommitDetails.cs index 8db1bbe..6c286fd 100644 --- a/GitTools/Analysis/SimpleAnalyzer/SimpleCommitDetails.cs +++ b/GitTools/Analysis/SimpleAnalyzer/SimpleCommitDetails.cs @@ -83,68 +83,74 @@ public SimpleCommitDetails(String repoPathOrUrl, Commit commit) [CsvColumn(FieldIndex = 6, OutputFormat = "yyyy-MM-dd HH:MM:ss")] public DateTime CommitterTime => this.commit.Committer.When.UtcDateTime; - [CsvColumn(FieldIndex = 9)] + [CsvColumn(FieldIndex = 7)] + public Int64 AuthorTimeUnixEpochMilliSecs => this.commit.Author.When.ToUnixTimeMilliseconds(); + + [CsvColumn(FieldIndex = 8)] + public Int64 CommitterTimeUnixEpochMilliSecs => this.commit.Committer.When.ToUnixTimeMilliseconds(); + + [CsvColumn(FieldIndex = 11)] public String AuthorEmail => this.commit.Author.Email; - [CsvColumn(FieldIndex = 10)] + [CsvColumn(FieldIndex = 12)] public String CommitterEmail => this.commit.Committer.Email; /// /// This is a boolean field but we use 0/1 for compatibility reasons. /// - [CsvColumn(FieldIndex = 13)] + [CsvColumn(FieldIndex = 15)] public UInt32 IsInitialCommit { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 14)] + [CsvColumn(FieldIndex = 16)] public UInt32 IsMergeCommit { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 15)] + [CsvColumn(FieldIndex = 17)] public UInt32 NumberOfParentCommits { get; protected internal set; } = 0u; - [CsvColumn(FieldIndex = 16)] + [CsvColumn(FieldIndex = 18)] public String ParentCommitSHA1s { get; protected internal set; } = String.Empty; #region Keywords - [CsvColumn(FieldIndex = 39)] + [CsvColumn(FieldIndex = 41)] public UInt32 KW_add => this.keywords.KW_add; - [CsvColumn(FieldIndex = 40)] + [CsvColumn(FieldIndex = 42)] public UInt32 KW_allow => this.keywords.KW_allow; - [CsvColumn(FieldIndex = 41)] + [CsvColumn(FieldIndex = 43)] public UInt32 KW_bug => this.keywords.KW_bug; - [CsvColumn(FieldIndex = 42)] + [CsvColumn(FieldIndex = 44)] public UInt32 KW_chang => this.keywords.KW_chang; - [CsvColumn(FieldIndex = 43)] + [CsvColumn(FieldIndex = 45)] public UInt32 KW_error => this.keywords.KW_error; - [CsvColumn(FieldIndex = 44)] + [CsvColumn(FieldIndex = 46)] public UInt32 KW_fail => this.keywords.KW_fail; - [CsvColumn(FieldIndex = 45)] + [CsvColumn(FieldIndex = 47)] public UInt32 KW_fix => this.keywords.KW_fix; - [CsvColumn(FieldIndex = 46)] + [CsvColumn(FieldIndex = 48)] public UInt32 KW_implement => this.keywords.KW_implement; - [CsvColumn(FieldIndex = 47)] + [CsvColumn(FieldIndex = 49)] public UInt32 KW_improv => this.keywords.KW_improv; - [CsvColumn(FieldIndex = 48)] + [CsvColumn(FieldIndex = 50)] public UInt32 KW_issu => this.keywords.KW_issu; - [CsvColumn(FieldIndex = 49)] + [CsvColumn(FieldIndex = 51)] public UInt32 KW_method => this.keywords.KW_method; - [CsvColumn(FieldIndex = 50)] + [CsvColumn(FieldIndex = 52)] public UInt32 KW_new => this.keywords.KW_new; - [CsvColumn(FieldIndex = 51)] + [CsvColumn(FieldIndex = 53)] public UInt32 KW_npe => this.keywords.KW_npe; - [CsvColumn(FieldIndex = 52)] + [CsvColumn(FieldIndex = 54)] public UInt32 KW_refactor => this.keywords.KW_refactor; - [CsvColumn(FieldIndex = 53)] + [CsvColumn(FieldIndex = 55)] public UInt32 KW_remov => this.keywords.KW_remov; - [CsvColumn(FieldIndex = 54)] + [CsvColumn(FieldIndex = 56)] public UInt32 KW_report => this.keywords.KW_report; - [CsvColumn(FieldIndex = 55)] + [CsvColumn(FieldIndex = 57)] public UInt32 KW_set => this.keywords.KW_set; - [CsvColumn(FieldIndex = 56)] + [CsvColumn(FieldIndex = 58)] public UInt32 KW_support => this.keywords.KW_support; - [CsvColumn(FieldIndex = 57)] + [CsvColumn(FieldIndex = 59)] public UInt32 KW_test => this.keywords.KW_test; - [CsvColumn(FieldIndex = 58)] + [CsvColumn(FieldIndex = 60)] public UInt32 KW_use => this.keywords.KW_use; #endregion @@ -153,7 +159,7 @@ public SimpleCommitDetails(String repoPathOrUrl, Commit commit) /// If using the , this will be /// the commit's short message. /// - [CsvColumn(FieldIndex = 8)] + [CsvColumn(FieldIndex = 9)] public virtual String Message => RegexNewLines.Replace(this.commit.MessageShort, " ").Replace('"', ' ').Trim(); #endregion diff --git a/GitTools/App.config b/GitTools/App.config index c99c01c..4a48133 100644 --- a/GitTools/App.config +++ b/GitTools/App.config @@ -36,7 +36,7 @@ - + @@ -56,7 +56,7 @@ - + @@ -84,7 +84,7 @@ - + @@ -102,6 +102,10 @@ + + + + \ No newline at end of file diff --git a/GitTools/GitTools.csproj b/GitTools/GitTools.csproj index 812ff01..85126c6 100644 --- a/GitTools/GitTools.csproj +++ b/GitTools/GitTools.csproj @@ -42,14 +42,14 @@ ..\packages\Antlr3.Runtime.3.5.1\lib\net40-client\Antlr3.Runtime.dll - - ..\packages\BouncyCastle.1.8.9\lib\BouncyCastle.Crypto.dll + + ..\packages\Portable.BouncyCastle.1.9.0\lib\net40\BouncyCastle.Crypto.dll ..\packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll - - ..\packages\Google.Protobuf.3.20.1\lib\net45\Google.Protobuf.dll + + ..\packages\Google.Protobuf.3.21.7\lib\net45\Google.Protobuf.dll ..\packages\Iesi.Collections.4.0.4\lib\net461\Iesi.Collections.dll @@ -90,8 +90,8 @@ ..\packages\Microsoft.Extensions.Logging.6.0.0\lib\net461\Microsoft.Extensions.Logging.dll - - ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.1\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll + + ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.2\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll ..\packages\Microsoft.Extensions.Options.6.0.0\lib\net461\Microsoft.Extensions.Options.dll @@ -99,14 +99,14 @@ ..\packages\Microsoft.Extensions.Primitives.6.0.0\lib\net461\Microsoft.Extensions.Primitives.dll - - ..\packages\MySql.Data.8.0.29\lib\net452\MySql.Data.dll + + ..\packages\MySql.Data.8.0.30\lib\net452\MySql.Data.dll ..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll - ..\packages\NHibernate.5.3.12\lib\net461\NHibernate.dll + ..\packages\NHibernate.5.3.13\lib\net461\NHibernate.dll ..\packages\Remotion.Linq.2.2.0\lib\net45\Remotion.Linq.dll @@ -114,8 +114,8 @@ ..\packages\Remotion.Linq.EagerFetching.2.2.0\lib\net45\Remotion.Linq.EagerFetching.dll - - ..\packages\SSH.NET.2020.0.1\lib\net40\Renci.SshNet.dll + + ..\packages\SSH.NET.2020.0.2\lib\net40\Renci.SshNet.dll @@ -138,8 +138,8 @@ - - ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll @@ -163,10 +163,10 @@ - ..\packages\MySql.Data.8.0.29\lib\net452\Ubiety.Dns.Core.dll + ..\packages\MySql.Data.8.0.30\lib\net452\Ubiety.Dns.Core.dll - ..\packages\MySql.Data.8.0.29\lib\net452\ZstdNet.dll + ..\packages\MySql.Data.8.0.30\lib\net452\ZstdNet.dll @@ -201,7 +201,7 @@ This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + \ No newline at end of file diff --git a/GitTools/Program.cs b/GitTools/Program.cs index feb64bd..785e04f 100644 --- a/GitTools/Program.cs +++ b/GitTools/Program.cs @@ -63,7 +63,7 @@ public static BaseLogger CreateLogger() }; } - private static BaseLogger logger = CreateLogger(); + private static readonly BaseLogger logger = CreateLogger(); /// /// Main entry point for application. @@ -156,7 +156,7 @@ static void Main(string[] args) } var span = new GitCommitSpan( - repo, options.Since, options.Until, options.Limit, sha1IDs); + repo, options.Since, options.Until, options.Limit, sha1IDs, options.SinceUseDate, options.UntilUseDate); if (span.SHA1Filter.Count > 0) { logger.LogTrace($"Using the following commit-IDs for the {nameof(GitCommitSpan)}: {String.Join(", ", span.SHA1Filter)}"); @@ -222,11 +222,11 @@ static void Main(string[] args) logger.LogInformation("Analysis done, attempting to write to CSV.."); if (options.AnalysisType == AnalysisType.Simple) { - csvc.Write(details.Cast(), writer, outd); + csvc.Write(details.Cast().OrderBy(c => c.CommitterTime), writer, outd); } else if (options.AnalysisType == AnalysisType.Extended) { - csvc.Write(details.Cast(), writer, outd); + csvc.Write(details.Cast().OrderBy(c => c.CommitterTime), writer, outd); } logger.LogInformation($"Wrote {details.Count} rows to file {options.OutputFile}."); @@ -362,9 +362,17 @@ internal class CommandLineOptions [Option('s', "since", Required = false, HelpText = "Optional. Analyze data since a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Since { get; set; } + [Option("since-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using a since-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate SinceUseDate { get; set; } + [Option('u', "until", Required = false, HelpText = "Optional. Analyze data until (inclusive) a certain date or SHA1. The required format for a date/time is 'yyyy-MM-dd HH:mm'. If using a hash, at least 3 characters are required.")] public String Until { get; set; } + [Option("until-use-date", Required = false, DefaultValue = SinceUntilUseDate.Committer, HelpText = "Optional. If using an until-date to delimit the range of commits, it can either be extracted from the " + nameof(SinceUntilUseDate.Author) + " or the " + nameof(SinceUntilUseDate.Committer) + ".")] + [JsonConverter(typeof(StringEnumConverter))] + public SinceUntilUseDate UntilUseDate { get; set; } + [Option('a', "analysis-type", Required = false, DefaultValue = AnalysisType.Extended, HelpText = "Optional. The type of analysis to run. Allowed values are " + nameof(AnalysisType.Simple) + " and " + nameof(AnalysisType.Extended) + ". The extended analysis extracts all supported properties of any Git-repository.")] [JsonConverter(typeof(StringEnumConverter))] public AnalysisType AnalysisType { get; set; } = AnalysisType.Extended; diff --git a/GitTools/packages.config b/GitTools/packages.config index 3fcca29..47b4376 100644 --- a/GitTools/packages.config +++ b/GitTools/packages.config @@ -3,7 +3,7 @@ - + @@ -18,20 +18,21 @@ - + - + - + + - + - + diff --git a/Util/Extensions/RepositoryExtensions.cs b/Util/Extensions/RepositoryExtensions.cs index 12806e5..01a284f 100644 --- a/Util/Extensions/RepositoryExtensions.cs +++ b/Util/Extensions/RepositoryExtensions.cs @@ -27,6 +27,9 @@ using System.Reflection; using System.Diagnostics; using Signature = LibGit2Sharp.Signature; +using Renci.SshNet.Common; +using System.Collections.Concurrent; +using System.Threading; namespace Util.Extensions { @@ -422,7 +425,7 @@ public static IDictionary - /// Similar to , this + /// Similar to , this /// method returns a mapping from each to a /// . More than one /// can point to the same . @@ -489,6 +492,102 @@ public static void Clear(this PatchEntryChanges pec, Boolean setStringBuilderNul } } + + /// + /// Collection of identical, yet differently located repositories. This is + /// useful for when concurrent, yet mutually exclusive operations on a single + /// need to be performed. + /// + public class RepositoryBundleCollection : LoanCollection + { + /// + /// A reference to the original that can + /// be used to add instances to this collection. + /// + public Repository Repository { get; private set; } + + /// + /// If true, then, on dispose of this collection, all items owned + /// by it will also be disposed by calling their method. + /// + public virtual Boolean DeleteClonedReposAfterwards { get; set; } + + /// + /// Creates a new bundle collection. + /// + /// + /// + public RepositoryBundleCollection(Repository repository, bool deleteClonedReposAfterwards = true) + : base() + { + this.Repository = repository; + this.DeleteClonedReposAfterwards = deleteClonedReposAfterwards; + } + + #region Dispose + private readonly Object disposeLock = new Object(); + /// + /// Disposes this collection and conditionally all its items, by deleting + /// the repositories on disk first and then disposing the related object. + /// Lastly, calls . + /// + /// + protected override sealed void Dispose(bool disposing) + { + lock (this.disposeLock) + { + if (!this.wasDisposed) + { + if (disposing) + { + foreach (var loanableItem in this) + { + try + { + if (this.DeleteClonedReposAfterwards) + { + var repo = loanableItem.Item; + var repoPath = repo.Info.WorkingDirectory; + repo.Dispose(); + new DirectoryInfo(repoPath).TryDelete(); + } + } + catch { } + } + } + + base.Dispose(disposing); + } + } + } + #endregion + } + + /// + /// Create a collection of identical repositories that are differently + /// located, so that mutually exclusive concurrent operations can be + /// carried out on each instance independently. Uses + /// to create identical instances. See and . + /// + /// + /// + /// + /// Defaults to true. Upon disposal of the collection, all the repositories will be destroyed. + /// + public static RepositoryBundleCollection CreateBundleCollection( + this Repository repository, String targetPath, UInt16 numInstances, Boolean deleteClonedReposAfterwards = true) + { + var rbc = new RepositoryBundleCollection( + repository, deleteClonedReposAfterwards: deleteClonedReposAfterwards); + + Parallel.ForEach(Enumerable.Range(0, numInstances), _ => + { + rbc.Add(repository.BundleAndCloneTo(targetPath)); + }); + + return rbc; + } + /// /// Bundles this to a single file (a git bundle), moves /// the file to the target-path and un-bundles (clones from) it. Then removes the diff --git a/Util/GitCommitSpan.cs b/Util/GitCommitSpan.cs index c61525b..9e68dee 100644 --- a/Util/GitCommitSpan.cs +++ b/Util/GitCommitSpan.cs @@ -26,6 +26,27 @@ namespace Util { + /// + /// When using since and/or until in a or other supporting + /// types, the can be extracted from either the author or the + /// commiter, as a has two signatures, + /// and . + /// + public enum SinceUntilUseDate + { + /// + /// Use the of when the commit was authored. + /// + Author = 1, + + /// + /// Use the of when then commit was incorporated (merged, rebased, + /// cherry-picked etc.). + /// + Committer = 2 + } + + /// /// Represents a segment that spans from a point in time or /// to another point in time or and thus represents a range @@ -82,6 +103,12 @@ public class GitCommitSpan : IDisposable, IEnumerable [JsonProperty(NullValueHandling = NullValueHandling.Include, Order = 4)] public String UntilCommitSha { get; private set; } + [JsonProperty(Order = 5)] + public SinceUntilUseDate SinceUseDate { get; private set; } = SinceUntilUseDate.Committer; + + [JsonProperty(Order = 6)] + public SinceUntilUseDate UntilUseDate { get; private set; } = SinceUntilUseDate.Committer; + [JsonIgnore] public String SinceAsString => this.SinceDateTime.HasValue ? this.SinceDateTime.ToString() : @@ -97,7 +124,7 @@ public class GitCommitSpan : IDisposable, IEnumerable /// with since/until, this can be a powerful tool to retrieve a certain amount /// of commits using offsets. Note that the limit is applied after the filter. /// - [JsonProperty(NullValueHandling = NullValueHandling.Include, Order = 5)] + [JsonProperty(NullValueHandling = NullValueHandling.Include, Order = 7)] public UInt32? Limit { get; private set; } [JsonIgnore] @@ -106,7 +133,7 @@ public class GitCommitSpan : IDisposable, IEnumerable /// A set of SHA1 that can be set to limit this even /// beyond any since/until constraints. Defaults to an empty set. /// - [JsonProperty(NullValueHandling = NullValueHandling.Include, Order = 6)] + [JsonProperty(NullValueHandling = NullValueHandling.Include, Order = 8)] public ReadOnlySet SHA1Filter => this.sha1Filter; /// @@ -140,13 +167,24 @@ public GitCommitSpan(Repository repository, Commit sinceCommit, Commit untilComm /// for . /// /// - public GitCommitSpan(Repository repository, String sinceDateTimeOrCommitSha = null, String untilDatetimeOrCommitSha = null, UInt32? limit = null, ISet sha1IDs = null) + /// + /// + public GitCommitSpan(Repository repository, String sinceDateTimeOrCommitSha = null, String untilDatetimeOrCommitSha = null, UInt32? limit = null, ISet sha1IDs = null, SinceUntilUseDate? sinceUseDate = null, SinceUntilUseDate? untilUseDate = null) { this.Repository = repository; this.Limit = limit; this.sha1Filter = new ReadOnlySet(new HashSet( (sha1IDs ?? Enumerable.Empty()).Select(v => v.Trim().ToLower()))); + if (sinceUseDate.HasValue) + { + this.SinceUseDate = sinceUseDate.Value; + } + if (untilUseDate.HasValue) + { + this.UntilUseDate = untilUseDate.Value; + } + var ic = CultureInfo.InvariantCulture; if (sinceDateTimeOrCommitSha == null) @@ -202,13 +240,21 @@ public GitCommitSpan(Repository repository, String sinceDateTimeOrCommitSha = nu ie = ie.Take((Int32)this.Limit.Value); } + Func authorSelector = c => c.Author.When, + committerSelector = c => c.Committer.When; + + var signatureSinceWhenSelector = this.SinceUseDate == SinceUntilUseDate.Author ? + authorSelector : committerSelector; + var signatureUntilWhenSelector = this.UntilUseDate == SinceUntilUseDate.Author ? + authorSelector : committerSelector; + var orderedOldToNew = new List(ie); var idxSince = orderedOldToNew.FindIndex(commit => { if (this.SinceDateTime.HasValue) { - return commit.Committer.When.UtcDateTime >= this.SinceDateTime; + return signatureSinceWhenSelector(commit).UtcDateTime >= this.SinceDateTime; } else if (this.SinceCommitSha != null) { @@ -219,7 +265,7 @@ public GitCommitSpan(Repository repository, String sinceDateTimeOrCommitSha = nu }); var idxUntil = this.UntilDateTime.HasValue ? - orderedOldToNew.TakeWhile(commit => commit.Committer.When.UtcDateTime <= this.UntilDateTime).Count() - 1 + orderedOldToNew.TakeWhile(commit => signatureUntilWhenSelector(commit).UtcDateTime <= this.UntilDateTime).Count() - 1 : orderedOldToNew.FindIndex(commit => commit.Sha.StartsWith(this.UntilCommitSha, StringComparison.InvariantCultureIgnoreCase)); diff --git a/Util/ISupportsExecutionPolicy.cs b/Util/ISupportsExecutionPolicy.cs index f6eecce..c39261f 100644 --- a/Util/ISupportsExecutionPolicy.cs +++ b/Util/ISupportsExecutionPolicy.cs @@ -14,10 +14,6 @@ /// --------------------------------------------------------------------------------- /// using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Util { diff --git a/Util/LoanableItem.cs b/Util/LoanableItem.cs new file mode 100644 index 0000000..18c9022 --- /dev/null +++ b/Util/LoanableItem.cs @@ -0,0 +1,273 @@ +using Renci.SshNet.Common; +using System; +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace Util +{ + /// + /// Describes a collection items can be loaned from and + /// returned to. A loaned item is given out as , + /// so that an implementing class can keep track of them. + /// + /// + public interface ILoanCollection : IEnumerable>, IDisposable + { + /// + /// Return (or add a new) item to the collection, making it available + /// for future loans. + /// + /// + /// + ILoanCollection Return(ILoanableItem item); + + /// + /// Loan an item. This returns an that + /// this collection can use to track loans + /// + /// + ILoanableItem Loan(); + } + + /// + /// Desc + /// + /// + public interface ILoanableItem : IDisposable + { + /// + /// The item that was loaned. + /// + T Item { get; } + } + + /// + /// A simple implementation of that implements + /// a dispose-pattern so that it returns itself after using it. + /// + /// + public class LoanableItem : ILoanableItem, IEquatable> + { + /// + /// + /// + public T Item { get; protected internal set; } + + private readonly ILoanCollection owningCollection; + + /// + /// Create a new loaned item and keep track of the collection it + /// belongs to. + /// + /// + /// + public LoanableItem(T item, ILoanCollection owningCollection) + { + this.Item = item; + this.owningCollection = owningCollection; + } + + #region Dispose + private bool wasDisposed = false; + /// + /// Returns this to the owning collection. + /// + /// + protected virtual void Dispose(bool disposing) + { + if (!wasDisposed) + { + if (disposing) + { + this.owningCollection.Return(this); + } + + wasDisposed = true; + } + } + + /// + /// Does not dispose this , but rather returns + /// this instance to the owning collection. + /// + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + #endregion + + #region Equals + /// + /// Calls . + /// + /// + /// + public override bool Equals(object obj) + { + return this.Equals(obj as LoanableItem); + } + + /// + /// Uses the hash codes of the wrapped item and the owning collection + /// to compute a hash code. + /// + /// + public override int GetHashCode() + { + if (EqualityComparer.Default.Equals(this.Item, default)) + { + return 0 ^ this.owningCollection.GetHashCode(); + } + + return this.Item.GetHashCode() ^ this.owningCollection.GetHashCode(); + } + + /// + /// Returns true if the wrapped items and owning collection are equal + /// to those of the other object. + /// + /// + /// + public bool Equals(LoanableItem other) + { + return other is LoanableItem + && Object.Equals(this.Item, other.Item) + && Object.Equals(this.owningCollection, other.owningCollection); + } + #endregion + } + + + /// + /// Implementation of an that uses a + /// queue for its items internally, as well as a semaphore (and thus + /// blocking behavior) for giving out items. + /// + /// + public class LoanCollection : ILoanCollection + { + /// + /// Returns the amount of items currently available for loan. + /// + public int Count => this.items.Count; + + private readonly SemaphoreLight semaphore; + + /// + /// Queue used for loaning items. + /// + protected readonly ConcurrentQueue items; + + /// + /// Creates a new . + /// + public LoanCollection() + { + this.semaphore = new SemaphoreLight(initialCount: 0); + this.items = new ConcurrentQueue(); + } + + /// + /// Add a new item to this collection that can be loaned. + /// + /// + /// + public virtual ILoanCollection Add(T item) + { + this.items.Enqueue(item); + this.semaphore.Release(); + return this; + } + + /// + /// Return a previously loaned item. This method will not check + /// whether the item was previously loaned from this collection, + /// but rather unwrap it and call . + /// + /// + /// + public virtual ILoanCollection Return(ILoanableItem item) + { + return this.Add(item.Item); + } + + /// + /// Loan an item. This method is subject to availability of items + /// and will block until an item is available. If you want to avoid + /// blocking, you can check the availability with . + /// Loaning and returning items is subject to a FIFO queue. + /// + /// + public virtual ILoanableItem Loan() + { + this.semaphore.Wait(); + this.items.TryDequeue(out T item); + return new LoanableItem(item, this); + } + + private readonly Object enumerableLock = new Object(); + + /// + /// Can be used to enumerate this collection. However, enumerating + /// results in loaning each item, so that after full enumeration, this + /// collection is empty. Furthermore, this is a synchronized method, + /// so that no two enumerators can obtain loans simultaneously, i.e., + /// after full enumeration of this collection, another enumerator can + /// enter. + /// + /// + public virtual IEnumerator> GetEnumerator() + { + lock (this.enumerableLock) + { + while (this.semaphore.CurrentCount > 0) + { + yield return this.Loan(); + } + } + } + + /// + /// Returns . + /// + /// + IEnumerator IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + + + #region IDispose + protected bool wasDisposed = false; + + /// + /// Disposes the underlying , but does not + /// dispose the items in the collection. + /// + /// + protected virtual void Dispose(bool disposing) + { + if (!this.wasDisposed) + { + if (disposing) + { + while (this.items.TryDequeue(out T _)); + this.semaphore.Dispose(); + } + + this.wasDisposed = true; + } + } + + public virtual void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + #endregion + } +} diff --git a/Util/Util.csproj b/Util/Util.csproj index f36ed18..3168902 100644 --- a/Util/Util.csproj +++ b/Util/Util.csproj @@ -83,8 +83,8 @@ ..\packages\Microsoft.Extensions.Logging.6.0.0\lib\net461\Microsoft.Extensions.Logging.dll - - ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.1\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll + + ..\packages\Microsoft.Extensions.Logging.Abstractions.6.0.2\lib\net461\Microsoft.Extensions.Logging.Abstractions.dll ..\packages\Microsoft.Extensions.Options.6.0.0\lib\net461\Microsoft.Extensions.Options.dll @@ -100,7 +100,7 @@ ..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll - ..\packages\NHibernate.5.3.12\lib\net461\NHibernate.dll + ..\packages\NHibernate.5.3.13\lib\net461\NHibernate.dll ..\packages\Remotion.Linq.2.2.0\lib\net45\Remotion.Linq.dll @@ -108,8 +108,8 @@ ..\packages\Remotion.Linq.EagerFetching.2.2.0\lib\net45\Remotion.Linq.EagerFetching.dll - - ..\packages\SSH.NET.2020.0.1\lib\net40\Renci.SshNet.dll + + ..\packages\SSH.NET.2020.0.2\lib\net40\Renci.SshNet.dll @@ -134,14 +134,14 @@ True - - ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\lib\net46\System.Data.SQLite.dll + + ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.116.0\lib\net46\System.Data.SQLite.dll - - ..\packages\System.Data.SQLite.EF6.1.0.115.5\lib\net46\System.Data.SQLite.EF6.dll + + ..\packages\System.Data.SQLite.EF6.1.0.116.0\lib\net46\System.Data.SQLite.EF6.dll - - ..\packages\System.Data.SQLite.Linq.1.0.115.5\lib\net46\System.Data.SQLite.Linq.dll + + ..\packages\System.Data.SQLite.Linq.1.0.116.0\lib\net46\System.Data.SQLite.Linq.dll ..\packages\System.Diagnostics.DiagnosticSource.6.0.0\lib\net461\System.Diagnostics.DiagnosticSource.dll @@ -167,8 +167,8 @@ ..\packages\System.IO.FileSystem.Primitives.4.3.0\lib\net46\System.IO.FileSystem.Primitives.dll True - - ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll ..\packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll @@ -257,6 +257,7 @@ + @@ -279,11 +280,11 @@ - - + + - - + + \ No newline at end of file diff --git a/Util/app.config b/Util/app.config index 36b9af8..6c1966b 100644 --- a/Util/app.config +++ b/Util/app.config @@ -32,7 +32,7 @@ - + @@ -52,7 +52,7 @@ - + diff --git a/Util/packages.config b/Util/packages.config index 175dcb1..1b53185 100644 --- a/Util/packages.config +++ b/Util/packages.config @@ -15,18 +15,18 @@ - + - + - + - - + + @@ -34,10 +34,10 @@ - - - - + + + + @@ -51,7 +51,7 @@ - + diff --git a/UtilTests/UtilTests.csproj b/UtilTests/UtilTests.csproj index 70c42f8..83216a0 100644 --- a/UtilTests/UtilTests.csproj +++ b/UtilTests/UtilTests.csproj @@ -61,7 +61,7 @@ ..\packages\MSTest.TestFramework.2.2.10\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll - ..\packages\NHibernate.5.3.12\lib\net461\NHibernate.dll + ..\packages\NHibernate.5.3.13\lib\net461\NHibernate.dll ..\packages\Remotion.Linq.2.2.0\lib\net45\Remotion.Linq.dll @@ -69,21 +69,21 @@ ..\packages\Remotion.Linq.EagerFetching.2.2.0\lib\net45\Remotion.Linq.EagerFetching.dll - - ..\packages\SSH.NET.2020.0.1\lib\net40\Renci.SshNet.dll + + ..\packages\SSH.NET.2020.0.2\lib\net40\Renci.SshNet.dll - - ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.115.5\lib\net46\System.Data.SQLite.dll + + ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.116.0\lib\net46\System.Data.SQLite.dll - - ..\packages\System.Data.SQLite.EF6.1.0.115.5\lib\net46\System.Data.SQLite.EF6.dll + + ..\packages\System.Data.SQLite.EF6.1.0.116.0\lib\net46\System.Data.SQLite.EF6.dll - - ..\packages\System.Data.SQLite.Linq.1.0.115.5\lib\net46\System.Data.SQLite.Linq.dll + + ..\packages\System.Data.SQLite.Linq.1.0.116.0\lib\net46\System.Data.SQLite.Linq.dll @@ -113,9 +113,9 @@ - + - + \ No newline at end of file diff --git a/UtilTests/app.config b/UtilTests/app.config index aee3abc..c0041d8 100644 --- a/UtilTests/app.config +++ b/UtilTests/app.config @@ -32,7 +32,7 @@ - + @@ -52,7 +52,7 @@ - + diff --git a/UtilTests/packages.config b/UtilTests/packages.config index 207e2e9..4df593a 100644 --- a/UtilTests/packages.config +++ b/UtilTests/packages.config @@ -5,13 +5,13 @@ - + - - - - - - + + + + + + \ No newline at end of file diff --git a/readme.md b/readme.md index fed7eb2..92854d4 100644 --- a/readme.md +++ b/readme.md @@ -9,11 +9,19 @@ It was developed during the research phase of the short technical paper and post To build the application, restore all _nuget_ packages and simply rebuild all projects. Run `GitDensity.exe`, which has an exhaustive command line interface for analyzing repositories. This implementation also includes a reimplementation of `git-hours` [2], runnable using `GitHours.exe` (with a similar command line interface). +There are also separate command line tools for extracting metrics (`GitMetrics.exe`) and smaller utility that unites a few stand-alone commands (`GitTools.exe`, see below). ## Requirement of external tools -This application relies on an external executable to run clone detection. Currently, it uses a local version of Softwerk's clone detection service [3]. To obtain a copy of this tool, please contact welf.lowe@lnu.se. -As for `git-metrics`, the application relies on another tool that supports currently obtaining software metrics from Java applications. Please contact me if you intend to use Git Metrics and require the tool. +This application relies on an external executable to run clone detection. Currently, it uses a local version of Softwerk's clone detection service [3]. +To obtain a copy free for academic use of this tool, please contact welf.lowe@lnu.se. + +You are not required to use the clone detection in order to obtain a notion fo source code density. In order to obtain a rough notion of it, you may use `git-tools` which will extract a ratio of net-lines to gross-lines as density. +The clone detection used in `git-density`, however, also computes a string similarity which will yield a most-precise approximation of the source code density. + +As for `git-metrics`, the application relies on another tool that supports currently obtaining software metrics from Java applications. +Metrics are obtained by building the application (for each commit). +Please contact me if you intend to use Git Metrics and require the tool. The tool is free for academic use. # Structure of the applications @@ -33,27 +41,30 @@ Git Density is a solution that currently features these three applications: * Has its own command-line interface and supports online/offline repos and parallelization. * Supports two methods currently: _Simple_ and _Extended_ (default) extraction. * Does not require tools for clone-detection or metrics, as these are not extracted. - * Extracts __38__ features (__13__ in _Simple_-mode): `"SHA1", "RepoPathOrUrl", "AuthorName", "CommitterName", "AuthorTime", "CommitterTime", "Message", "AuthorEmail", "CommitterEmail", "IsInitialCommit", "IsMergeCommit", "NumberOfParentCommits", "ParentCommitSHA1s"` __plus 25 in extended:__ `"MinutesSincePreviousCommit", "AuthorNominalLabel", "CommitterNominalLabel", "NumberOfFilesAdded", "NumberOfFilesAddedNet", "NumberOfLinesAddedByAddedFiles", "NumberOfLinesAddedByAddedFilesNet", "NumberOfFilesDeleted", "NumberOfFilesDeletedNet", "NumberOfLinesDeletedByDeletedFiles", "NumberOfLinesDeletedByDeletedFilesNet", "NumberOfFilesModified", "NumberOfFilesModifiedNet", "NumberOfFilesRenamed", "NumberOfFilesRenamedNet", "NumberOfLinesAddedByModifiedFiles", "NumberOfLinesAddedByModifiedFilesNet", "NumberOfLinesDeletedByModifiedFiles", "NumberOfLinesDeletedByModifiedFilesNet", "NumberOfLinesAddedByRenamedFiles", "NumberOfLinesAddedByRenamedFilesNet", "NumberOfLinesDeletedByRenamedFiles", "NumberOfLinesDeletedByRenamedFilesNet", "Density", "AffectedFilesRatioNet"` + * Extracts __58__ features (__13__ features + counts for __20__ keywords (see [5]) in _Simple_-mode): `"SHA1", "RepoPathOrUrl", "AuthorName", "CommitterName", "AuthorTime", "CommitterTime", "Message", "AuthorEmail", "CommitterEmail", "IsInitialCommit", "IsMergeCommit", "NumberOfParentCommits", "ParentCommitSHA1s"` __plus 25 in extended:__ `"MinutesSincePreviousCommit", "AuthorNominalLabel", "CommitterNominalLabel", "NumberOfFilesAdded", "NumberOfFilesAddedNet", "NumberOfLinesAddedByAddedFiles", "NumberOfLinesAddedByAddedFilesNet", "NumberOfFilesDeleted", "NumberOfFilesDeletedNet", "NumberOfLinesDeletedByDeletedFiles", "NumberOfLinesDeletedByDeletedFilesNet", "NumberOfFilesModified", "NumberOfFilesModifiedNet", "NumberOfFilesRenamed", "NumberOfFilesRenamedNet", "NumberOfLinesAddedByModifiedFiles", "NumberOfLinesAddedByModifiedFilesNet", "NumberOfLinesDeletedByModifiedFiles", "NumberOfLinesDeletedByModifiedFilesNet", "NumberOfLinesAddedByRenamedFiles", "NumberOfLinesAddedByRenamedFilesNet", "NumberOfLinesDeletedByRenamedFiles", "NumberOfLinesDeletedByRenamedFilesNet", "Density", "AffectedFilesRatioNet"` All applications can be run standalone, but may also be included as references, as they all feature a public API. -## Caveats -If using `MySQL`, the latest 5.7.x GA-releases work, while some of the 8.x versions appear to cause problems in conjunction with Fluent NHibernate (this should be fixed in version 2020.1). You may also use other types of databases, as Git Density supports these: `MsSQL2000`, `MsSQL2005`, `MsSQL2008`, `MsSQL2012`, `MySQL`, `Oracle10`, `Oracle9`, `PgSQL81`, `PgSQL82`, `SQLite`, `SQLiteTemp` (temporary database that is discarded after the analysis, mainly for testing). + +## About Databases + +You may also use other types of databases, as Git Density supports these: `MsSQL2000`, `MsSQL2005`, `MsSQL2008`, `MsSQL2012`, `MySQL`, `Oracle10`, `Oracle9`, `PgSQL81`, `PgSQL82`, `SQLite`, `SQLiteTemp` (temporary database that is discarded after the analysis, mainly for testing). ___ # Citing Please use the following BibTeX to cite __`GitDensity`__: +
 @article{honel2020gitdensity,
-  title={Git Density (2020.1): Analyze git repositories to extract the Source Code Density and other Commit Properties},
+  title={Git Density (2022.10): Analyze git repositories to extract the Source Code Density and other Commit Properties},
   DOI={10.5281/zenodo.2565238},
   url={https://doi.org/10.5281/zenodo.2565238},
   publisher={Zenodo},
   author={Sebastian Hönel},
-  year={2020},
-  month={Jan},
+  year={2022},
+  month={Oct},
   abstractNote={Git Density (git-density) is a tool to analyze git-repositories with the goal of detecting the source code density. It was developed during the research phase of the short technical paper and poster "A changeset-based approach to assess source code density and developer efficacy" and has since been extended to support extended analyses.},
 }
 
@@ -69,3 +80,5 @@ ___ [3] QTools Clone Detection. http://qtools.se/ [4] Hönel, S., Ericsson, M., Löwe, W. and Wingkvist, A., 2019. Importance and Aptitude of Source code Density for Commit Classification into Maintenance Activities. In The 19th IEEE International Conference on Software Quality, Reliability, and Security. + +[5] Levin, S. and Yehudai, A., 2017, November. Boosting automatic commit classification into maintenance activities by utilizing source code changes. In Proceedings of the 13th International Conference on Predictive Models and Data Analytics in Software Engineering (pp. 97-106).