From eb442fc0e55169a31be542e632ecbfb6a038dc33 Mon Sep 17 00:00:00 2001 From: Amy Palamountain Date: Thu, 26 Feb 2015 16:39:29 -0800 Subject: [PATCH 1/4] Added custom filter support --- LibGit2Sharp.Tests/FilterFixture.cs | 271 ++++++++++++++++++ .../FilterSubstitutionCipherFixture.cs | 229 +++++++++++++++ LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj | 3 + .../TestHelpers/SubstitutionCipherFilter.cs | 49 ++++ LibGit2Sharp/Core/GitFilter.cs | 103 +++++++ LibGit2Sharp/Core/NativeMethods.cs | 22 +- LibGit2Sharp/Core/Proxy.cs | 68 +++++ LibGit2Sharp/Filter.cs | 176 ++++++++++++ LibGit2Sharp/FilterAttributeEntry.cs | 47 +++ LibGit2Sharp/FilterMode.cs | 20 ++ LibGit2Sharp/FilterRegistration.cs | 34 +++ LibGit2Sharp/FilterSource.cs | 51 ++++ LibGit2Sharp/GlobalSettings.cs | 34 +++ LibGit2Sharp/LibGit2Sharp.csproj | 6 + 14 files changed, 1112 insertions(+), 1 deletion(-) create mode 100644 LibGit2Sharp.Tests/FilterFixture.cs create mode 100644 LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs create mode 100644 LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs create mode 100644 LibGit2Sharp/Core/GitFilter.cs create mode 100644 LibGit2Sharp/Filter.cs create mode 100644 LibGit2Sharp/FilterAttributeEntry.cs create mode 100644 LibGit2Sharp/FilterMode.cs create mode 100644 LibGit2Sharp/FilterRegistration.cs create mode 100644 LibGit2Sharp/FilterSource.cs diff --git a/LibGit2Sharp.Tests/FilterFixture.cs b/LibGit2Sharp.Tests/FilterFixture.cs new file mode 100644 index 000000000..776e9c2b2 --- /dev/null +++ b/LibGit2Sharp.Tests/FilterFixture.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; +using System.IO; +using LibGit2Sharp.Tests.TestHelpers; +using Xunit; + +namespace LibGit2Sharp.Tests +{ + public class FilterFixture : BaseFixture + { + private const int GitPassThrough = -30; + + readonly Func successCallback = (reader, writer) => 0; + + private const string FilterName = "the-filter"; + readonly List attributes = new List { new FilterAttributeEntry("test") }; + + [Fact] + public void CanRegisterFilterWithSingleAttribute() + { + var filter = new EmptyFilter(FilterName, attributes); + Assert.Equal( attributes , filter.Attributes); + } + + [Fact] + public void CanRegisterAndUnregisterTheSameFilter() + { + var filter = new EmptyFilter(FilterName + 1, attributes); + + var registration = GlobalSettings.RegisterFilter(filter); + GlobalSettings.DeregisterFilter(registration); + + var secondRegistration = GlobalSettings.RegisterFilter(filter); + GlobalSettings.DeregisterFilter(secondRegistration); + } + + [Fact] + public void CanRegisterAndDeregisterAfterGarbageCollection() + { + var filter = new EmptyFilter(FilterName + 2, attributes); + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + GC.Collect(); + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Fact] + public void SameFilterIsEqual() + { + var filter = new EmptyFilter(FilterName + 3, attributes); + Assert.Equal(filter, filter); + } + + [Fact] + public void InitCallbackNotMadeWhenFilterNeverUsed() + { + bool called = false; + Func initializeCallback = () => + { + called = true; + return 0; + }; + + var filter = new FakeFilter(FilterName + 11, attributes, + successCallback, + successCallback, + initializeCallback); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + Assert.False(called); + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Fact] + public void InitCallbackMadeWhenUsingTheFilter() + { + bool called = false; + Func initializeCallback = () => + { + called = true; + return 0; + }; + + var filter = new FakeFilter(FilterName + 12, attributes, + successCallback, + successCallback, + initializeCallback); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + Assert.False(called); + + string repoPath = InitNewRepository(); + using (var repo = CreateTestRepository(repoPath)) + { + StageNewFile(repo); + Assert.True(called); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Fact] + public void WhenStagingFileApplyIsCalledWithCleanForCorrectPath() + { + string repoPath = InitNewRepository(); + bool called = false; + + Func clean = (reader, writer) => + { + called = true; + return GitPassThrough; + }; + var filter = new FakeFilter(FilterName + 15, attributes, clean); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + using (var repo = CreateTestRepository(repoPath)) + { + StageNewFile(repo); + Assert.True(called); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Fact] + public void CleanFilterWritesOutputToObjectTree() + { + const string decodedInput = "This is a substitution cipher"; + const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; + + string repoPath = InitNewRepository(); + + Func cleanCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; + + var filter = new FakeFilter(FilterName + 16, attributes, cleanCallback); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + using (var repo = CreateTestRepository(repoPath)) + { + FileInfo expectedFile = StageNewFile(repo, decodedInput); + var commit = repo.Commit("Clean that file"); + + var blob = (Blob)commit.Tree[expectedFile.Name].Target; + + var textDetected = blob.GetContentText(); + Assert.Equal(encodedInput, textDetected); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + + [Fact] + public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory() + { + const string decodedInput = "This is a substitution cipher"; + const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; + + const string branchName = "branch"; + string repoPath = InitNewRepository(); + + Func smudgeCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; + + var filter = new FakeFilter(FilterName + 17, attributes, null, smudgeCallback); + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + FileInfo expectedFile = CheckoutFileForSmudge(repoPath, branchName, encodedInput); + + string combine = Path.Combine(repoPath, "..", expectedFile.Name); + string readAllText = File.ReadAllText(combine); + Assert.Equal(decodedInput, readAllText); + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + private FileInfo CheckoutFileForSmudge(string repoPath, string branchName, string content) + { + FileInfo expectedPath; + using (var repo = CreateTestRepository(repoPath)) + { + StageNewFile(repo, content); + + repo.Commit("Initial commit"); + + expectedPath = CommitFileOnBranch(repo, branchName, content); + + repo.Checkout("master"); + + repo.Checkout(branchName); + } + return expectedPath; + } + + private static FileInfo CommitFileOnBranch(Repository repo, string branchName, String content) + { + var branch = repo.CreateBranch(branchName); + repo.Checkout(branch.Name); + + FileInfo expectedPath = StageNewFile(repo, content); + repo.Commit("Commit"); + return expectedPath; + } + + private static FileInfo StageNewFile(IRepository repo, string contents = "null") + { + string newFilePath = Touch(repo.Info.WorkingDirectory, Guid.NewGuid() + ".txt", contents); + var stageNewFile = new FileInfo(newFilePath); + repo.Stage(newFilePath); + return stageNewFile; + } + + private Repository CreateTestRepository(string path) + { + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + var repository = new Repository(path, repositoryOptions); + CreateAttributesFile(repository, "* filter=test"); + return repository; + } + + private static void CreateAttributesFile(IRepository repo, string attributeEntry) + { + Touch(repo.Info.WorkingDirectory, ".gitattributes", attributeEntry); + } + + class EmptyFilter : Filter + { + public EmptyFilter(string name, IEnumerable attributes) + : base(name, attributes) + { } + } + + class FakeFilter : Filter + { + private readonly Func cleanCallback; + private readonly Func smudgeCallback; + private readonly Func initCallback; + + public FakeFilter(string name, IEnumerable attributes, + Func cleanCallback = null, + Func smudgeCallback = null, + Func initCallback = null) + : base(name, attributes) + { + this.cleanCallback = cleanCallback; + this.smudgeCallback = smudgeCallback; + this.initCallback = initCallback; + } + + protected override int Clean(string path, Stream input, Stream output) + { + return cleanCallback != null ? cleanCallback(input, output) : base.Clean(path, input, output); + } + + protected override int Smudge(string path, Stream input, Stream output) + { + return smudgeCallback != null ? smudgeCallback(input, output) : base.Smudge(path, input, output); + } + + protected override int Initialize() + { + return initCallback != null ? initCallback() : base.Initialize(); + } + } + } +} diff --git a/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs b/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs new file mode 100644 index 000000000..5f429d001 --- /dev/null +++ b/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs @@ -0,0 +1,229 @@ +using System; +using System.Collections.Generic; +using System.IO; +using LibGit2Sharp.Tests.TestHelpers; +using Xunit; +using Xunit.Extensions; + +namespace LibGit2Sharp.Tests +{ + public class FilterSubstitutionCipherFixture : BaseFixture + { + [Fact] + public void SmugdeIsNotCalledForFileWhichDoesNotMatchAnAttributeEntry() + { + const string decodedInput = "This is a substitution cipher"; + const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; + + var attributes = new List { new FilterAttributeEntry("filter=rot13") }; + var filter = new SubstitutionCipherFilter("cipher-filter", attributes); + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string repoPath = InitNewRepository(); + string fileName = Guid.NewGuid() + ".rot13"; + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + using (var repo = new Repository(repoPath, repositoryOptions)) + { + CreateAttributesFile(repo, "*.rot13 filter=rot13"); + + var blob = CommitOnBranchAndReturnDatabaseBlob(repo, fileName, decodedInput); + var textDetected = blob.GetContentText(); + + Assert.Equal(encodedInput, textDetected); + Assert.Equal(1, filter.CleanCalledCount); + Assert.Equal(0, filter.SmudgeCalledCount); + + var branch = repo.CreateBranch("delete-files"); + repo.Checkout(branch.Name); + + DeleteFile(repo, fileName); + + repo.Checkout("master"); + + var fileContents = ReadTextFromFile(repo, fileName); + Assert.Equal(1, filter.SmudgeCalledCount); + Assert.Equal(decodedInput, fileContents); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Fact] + public void CorrectlyEncodesAndDecodesInput() + { + const string decodedInput = "This is a substitution cipher"; + const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; + + var attributes = new List { new FilterAttributeEntry("filter=rot13") }; + var filter = new SubstitutionCipherFilter("cipher-filter", attributes); + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string repoPath = InitNewRepository(); + string fileName = Guid.NewGuid() + ".rot13"; + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + using (var repo = new Repository(repoPath, repositoryOptions)) + { + CreateAttributesFile(repo, "*.rot13 filter=rot13"); + + var blob = CommitOnBranchAndReturnDatabaseBlob(repo, fileName, decodedInput); + var textDetected = blob.GetContentText(); + + Assert.Equal(encodedInput, textDetected); + Assert.Equal(1, filter.CleanCalledCount); + Assert.Equal(0, filter.SmudgeCalledCount); + + var branch = repo.CreateBranch("delete-files"); + repo.Checkout(branch.Name); + + DeleteFile(repo, fileName); + + repo.Checkout("master"); + + var fileContents = ReadTextFromFile(repo, fileName); + Assert.Equal(1, filter.SmudgeCalledCount); + Assert.Equal(decodedInput, fileContents); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Theory] + [InlineData("*.txt", ".bat", 0, 0)] + [InlineData("*.txt", ".txt", 1, 0)] + public void WhenStagedFileDoesNotMatchPathSpecFileIsNotFiltered(string pathSpec, string fileExtension, int cleanCount, int smudgeCount) + { + const string filterName = "filter=rot13"; + const string decodedInput = "This is a substitution cipher"; + string attributeFileEntry = string.Format("{0} {1}", pathSpec, filterName); + + var filterForAttributes = new List { new FilterAttributeEntry(filterName) }; + var filter = new SubstitutionCipherFilter("cipher-filter", filterForAttributes); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string repoPath = InitNewRepository(); + string fileName = Guid.NewGuid() + fileExtension; + + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + using (var repo = new Repository(repoPath, repositoryOptions)) + { + CreateAttributesFile(repo, attributeFileEntry); + + CommitOnBranchAndReturnDatabaseBlob(repo, fileName, decodedInput); + + Assert.Equal(cleanCount, filter.CleanCalledCount); + Assert.Equal(smudgeCount, filter.SmudgeCalledCount); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Theory] + [InlineData("filter=rot13", "*.txt filter=rot13", 1)] + [InlineData("filter=rot13", "*.txt filter=fake", 0)] + [InlineData("filter=rot13", "*.bat filter=rot13", 0)] + [InlineData("rot13", "*.txt filter=rot13", 1)] + [InlineData("rot13", "*.txt filter=fake", 0)] + [InlineData("fake", "*.txt filter=fake", 1)] + [InlineData("filter=fake", "*.txt filter=fake", 1)] + [InlineData("filter=fake", "*.bat filter=fake", 0)] + [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)] + public void CleanIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int cleanCount) + { + const string decodedInput = "This is a substitution cipher"; + + var filterForAttributes = new List { new FilterAttributeEntry(filterAttribute) }; + var filter = new SubstitutionCipherFilter("cipher-filter", filterForAttributes); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string repoPath = InitNewRepository(); + string fileName = Guid.NewGuid() + ".txt"; + + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + using (var repo = new Repository(repoPath, repositoryOptions)) + { + CreateAttributesFile(repo, attributeEntry); + + CommitOnBranchAndReturnDatabaseBlob(repo, fileName, decodedInput); + + Assert.Equal(cleanCount, filter.CleanCalledCount); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + [Theory] + [InlineData("filter=rot13", "*.txt filter=rot13", 1)] + [InlineData("filter=rot13", "*.txt filter=fake", 0)] + [InlineData("filter=rot13", "*.bat filter=rot13", 0)] + [InlineData("rot13", "*.txt filter=rot13", 1)] + [InlineData("rot13", "*.txt filter=fake", 0)] + [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)] + public void SmudgeIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int smudgeCount) + { + const string decodedInput = "This is a substitution cipher"; + + var filterForAttributes = new List { new FilterAttributeEntry(filterAttribute) }; + var filter = new SubstitutionCipherFilter("cipher-filter", filterForAttributes); + + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string repoPath = InitNewRepository(); + string fileName = Guid.NewGuid() + ".txt"; + + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + using (var repo = new Repository(repoPath, repositoryOptions)) + { + CreateAttributesFile(repo, attributeEntry); + + CommitOnBranchAndReturnDatabaseBlob(repo, fileName, decodedInput); + + var branch = repo.CreateBranch("delete-files"); + repo.Checkout(branch.Name); + + DeleteFile(repo, fileName); + + repo.Checkout("master"); + + Assert.Equal(smudgeCount, filter.SmudgeCalledCount); + } + + GlobalSettings.DeregisterFilter(filterRegistration); + + } + + private static string ReadTextFromFile(Repository repo, string fileName) + { + return File.ReadAllText(Path.Combine(repo.Info.WorkingDirectory, fileName)); + } + + private static void DeleteFile(Repository repo, string fileName) + { + File.Delete(Path.Combine(repo.Info.WorkingDirectory, fileName)); + repo.Stage(fileName); + repo.Commit("remove file"); + } + + private static Blob CommitOnBranchAndReturnDatabaseBlob(Repository repo, string fileName, string input) + { + Touch(repo.Info.WorkingDirectory, fileName, input); + repo.Stage(fileName); + + var commit = repo.Commit("new file"); + + var blob = (Blob)commit.Tree[fileName].Target; + return blob; + } + + private static void CreateAttributesFile(IRepository repo, string attributeEntry) + { + Touch(repo.Info.WorkingDirectory, ".gitattributes", attributeEntry); + } + } +} diff --git a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj index 5301de54e..5b39996a4 100644 --- a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj +++ b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj @@ -61,6 +61,7 @@ + @@ -103,6 +104,7 @@ + @@ -119,6 +121,7 @@ + diff --git a/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs b/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs new file mode 100644 index 000000000..ab29b0d77 --- /dev/null +++ b/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using System.IO; + +namespace LibGit2Sharp.Tests.TestHelpers +{ + public class SubstitutionCipherFilter : Filter + { + public int CleanCalledCount = 0; + public int SmudgeCalledCount = 0; + + public SubstitutionCipherFilter(string name, IEnumerable attributes) + : base(name, attributes) + { + } + + protected override int Clean(string path, Stream input, Stream output) + { + CleanCalledCount++; + return RotateByThirteenPlaces(input, output); + } + + protected override int Smudge(string path, Stream input, Stream output) + { + SmudgeCalledCount++; + return RotateByThirteenPlaces(input, output); + } + + public static int RotateByThirteenPlaces(Stream input, Stream output) + { + int value; + + while ((value = input.ReadByte()) != -1) + { + if ((value >= 'a' && value <= 'm') || (value >= 'A' && value <= 'M')) + { + value += 13; + } + else if ((value >= 'n' && value <= 'z') || (value >= 'N' && value <= 'Z')) + { + value -= 13; + } + + output.WriteByte((byte)value); + } + + return 0; + } + } +} diff --git a/LibGit2Sharp/Core/GitFilter.cs b/LibGit2Sharp/Core/GitFilter.cs new file mode 100644 index 000000000..285926bf2 --- /dev/null +++ b/LibGit2Sharp/Core/GitFilter.cs @@ -0,0 +1,103 @@ +using System; +using System.Runtime.InteropServices; +namespace LibGit2Sharp.Core +{ + /// + /// A git filter + /// + [StructLayout(LayoutKind.Sequential)] + internal class GitFilter + { + public uint version = 1; + + public IntPtr attributes; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_init_fn init; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_shutdown_fn shutdown; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_check_fn check; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_apply_fn apply; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_cleanup_fn cleanup; + + /* The libgit2 structure definition ends here. Subsequent fields are for libgit2sharp bookkeeping. */ + + /// + /// Initialize callback on filter + /// + /// Specified as `filter.initialize`, this is an optional callback invoked + /// before a filter is first used. It will be called once at most. + /// + /// If non-NULL, the filter's `initialize` callback will be invoked right + /// before the first use of the filter, so you can defer expensive + /// initialization operations (in case libgit2 is being used in a way that doesn't need the filter). + /// + public delegate int git_filter_init_fn(IntPtr filter); + + /// + /// Shutdown callback on filter + /// + /// Specified as `filter.shutdown`, this is an optional callback invoked + /// when the filter is unregistered or when libgit2 is shutting down. It + /// will be called once at most and should release resources as needed. + /// Typically this function will free the `git_filter` object itself. + /// + public delegate void git_filter_shutdown_fn(IntPtr filter); + + /// + /// Callback to decide if a given source needs this filter + /// Specified as `filter.check`, this is an optional callback that checks if filtering is needed for a given source. + /// + /// It should return 0 if the filter should be applied (i.e. success), GIT_PASSTHROUGH if the filter should + /// not be applied, or an error code to fail out of the filter processing pipeline and return to the caller. + /// + /// The `attr_values` will be set to the values of any attributes given in the filter definition. See `git_filter` below for more detail. + /// + /// The `payload` will be a pointer to a reference payload for the filter. This will start as NULL, but `check` can assign to this + /// pointer for later use by the `apply` callback. Note that the value should be heap allocated (not stack), so that it doesn't go + /// away before the `apply` callback can use it. If a filter allocates and assigns a value to the `payload`, it will need a `cleanup` + /// callback to free the payload. + /// + public delegate int git_filter_check_fn( + GitFilter gitFilter, IntPtr payload, IntPtr filterSource, IntPtr attributeValues); + + /// + /// Callback to actually perform the data filtering + /// + /// Specified as `filter.apply`, this is the callback that actually filters data. + /// If it successfully writes the output, it should return 0. Like `check`, + /// it can return GIT_PASSTHROUGH to indicate that the filter doesn't want to run. + /// Other error codes will stop filter processing and return to the caller. + /// + /// The `payload` value will refer to any payload that was set by the `check` callback. It may be read from or written to as needed. + /// + public delegate int git_filter_apply_fn( + GitFilter gitFilter, IntPtr payload, IntPtr gitBufTo, IntPtr gitBufFrom, IntPtr filterSource); + + /// + /// Callback to clean up after filtering has been applied. Specified as `filter.cleanup`, this is an optional callback invoked + /// after the filter has been applied. If the `check` or `apply` callbacks allocated a `payload` + /// to keep per-source filter state, use this callback to free that payload and release resources as required. + /// + public delegate void git_filter_cleanup_fn(IntPtr gitFilter, IntPtr payload); + } + /// + /// The file source being filtered + /// + [StructLayout(LayoutKind.Sequential)] + internal class GitFilterSource + { + public IntPtr repository; + + public IntPtr path; + + public GitOid oid; + } +} diff --git a/LibGit2Sharp/Core/NativeMethods.cs b/LibGit2Sharp/Core/NativeMethods.cs index be44a2f58..b05a1aeb9 100644 --- a/LibGit2Sharp/Core/NativeMethods.cs +++ b/LibGit2Sharp/Core/NativeMethods.cs @@ -191,6 +191,12 @@ internal static extern int git_branch_remote_name( RepositorySafeHandle repo, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(StrictUtf8Marshaler))] string canonical_branch_name); + [DllImport(libgit2)] + internal static extern int git_buf_grow(IntPtr buffer, UIntPtr targetSize); + + [DllImport(libgit2)] + internal static extern int git_buf_put(IntPtr buffer, IntPtr data, UIntPtr len); + [DllImport(libgit2)] internal static extern int git_remote_rename( ref GitStrArray problems, @@ -202,7 +208,6 @@ internal delegate int git_remote_rename_problem_cb( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(LaxUtf8NoCleanupMarshaler))] string problematic_refspec, IntPtr payload); - [DllImport(libgit2)] internal static extern int git_branch_upstream_name( GitBuf buf, @@ -506,6 +511,21 @@ internal static extern int git_diff_find_similar( [DllImport(libgit2)] internal static extern IntPtr git_diff_get_delta(DiffSafeHandle diff, UIntPtr idx); + [DllImport(libgit2)] + internal static extern int git_filter_register( + [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(StrictUtf8Marshaler))] string name, + IntPtr gitFilter, int priority); + + [DllImport(libgit2)] + internal static extern int git_filter_unregister( + [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(StrictUtf8Marshaler))]string name); + + [DllImport(libgit2)] + internal static extern int git_filter_source_mode(IntPtr source); + + [DllImport(libgit2)] + internal static extern void git_filter_free(IntPtr filterSafeHandle); + [DllImport(libgit2)] internal static extern int git_libgit2_features(); diff --git a/LibGit2Sharp/Core/Proxy.cs b/LibGit2Sharp/Core/Proxy.cs index ee7162377..2023eb681 100644 --- a/LibGit2Sharp/Core/Proxy.cs +++ b/LibGit2Sharp/Core/Proxy.cs @@ -276,6 +276,33 @@ public static string git_branch_upstream_name(RepositorySafeHandle handle, strin #region git_buf_ + public static void git_buf_grow(IntPtr gitBufPointer, ulong target_size) + { + using (ThreadAffinity()) + { + var res = NativeMethods.git_buf_grow(gitBufPointer, (UIntPtr)target_size); + Ensure.ZeroResult(res); + } + } + + public static void git_buf_put(IntPtr gitBufPointer, byte[] data, int offset, int count) + { + using (ThreadAffinity()) + { + unsafe + { + int res; + + fixed (byte* ptr = data) + { + res = NativeMethods.git_buf_put(gitBufPointer, (IntPtr)ptr, (UIntPtr)count); + } + + Ensure.ZeroResult(res); + } + } + } + public static void git_buf_free(GitBuf buf) { NativeMethods.git_buf_free(buf); @@ -806,6 +833,47 @@ public static GitDiffDelta git_diff_get_delta(DiffSafeHandle diff, int idx) #endregion + #region git_filter_ + + public static void git_filter_register(string name, IntPtr filter, int priority) + { + using (ThreadAffinity()) + { + int res = NativeMethods.git_filter_register(name, filter, priority); + if (res == (int)GitErrorCode.Exists) + { + var message = string.Format("A filter with the name '{0}' is already registered", name); + throw new EntryExistsException(message); + } + Ensure.ZeroResult(res); + } + } + + public static void git_filter_unregister(string name) + { + using (ThreadAffinity()) + { + int res = NativeMethods.git_filter_unregister(name); + Ensure.ZeroResult(res); + } + } + + public static FilterMode git_filter_source_mode(IntPtr filterSource) + { + var res = NativeMethods.git_filter_source_mode(filterSource); + return (FilterMode)res; + } + + public static void git_filter_free(IntPtr gitFilter) + { + using (ThreadAffinity()) + { + NativeMethods.git_filter_free(gitFilter); + } + } + + #endregion + #region git_graph_ public static Tuple git_graph_ahead_behind(RepositorySafeHandle repo, Commit first, Commit second) diff --git a/LibGit2Sharp/Filter.cs b/LibGit2Sharp/Filter.cs new file mode 100644 index 000000000..71947553f --- /dev/null +++ b/LibGit2Sharp/Filter.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using LibGit2Sharp.Core; + +namespace LibGit2Sharp +{ + /// + /// A filter is a way to execute code against a file as it moves to and from the git + /// repository and into the working directory. + /// + public abstract class Filter : IEquatable + { + private static readonly LambdaEqualityHelper equalityHelper = + new LambdaEqualityHelper(x => x.Name, x => x.Attributes); + + private readonly string name; + private readonly IEnumerable attributes; + + private readonly GitFilter gitFilter; + + /// + /// Initializes a new instance of the class. + /// And allocates the filter natively. + /// The unique name with which this filtered is registered with + /// A list of attributes which this filter applies to + /// + protected Filter(string name, IEnumerable attributes) + { + Ensure.ArgumentNotNullOrEmptyString(name, "name"); + Ensure.ArgumentNotNull(attributes, "attributes"); + + this.name = name; + this.attributes = attributes; + var attributesAsString = string.Join(",", this.attributes.Select(attr => attr.FilterDefinition)); + + gitFilter = new GitFilter + { + attributes = EncodingMarshaler.FromManaged(Encoding.UTF8, attributesAsString), + init = InitializeCallback, + }; + } + + /// + /// The name that this filter was registered with + /// + public string Name + { + get { return name; } + } + + /// + /// The filter filterForAttributes. + /// + public IEnumerable Attributes + { + get { return attributes; } + } + + /// + /// The marshalled filter + /// + internal GitFilter GitFilter + { + get { return gitFilter; } + } + + /// + /// Initialize callback on filter + /// + /// Specified as `filter.initialize`, this is an optional callback invoked + /// before a filter is first used. It will be called once at most. + /// + /// If non-NULL, the filter's `initialize` callback will be invoked right + /// before the first use of the filter, so you can defer expensive + /// initialization operations (in case the library is being used in a way + /// that doesn't need the filter. + /// + protected virtual int Initialize() + { + return 0; + } + + /// + /// Clean the input stream and write to the output stream. + /// + /// The path of the file being filtered + /// The git buf input reader + /// The git buf output writer + /// 0 if successful and to skip and pass through + protected virtual int Clean(string path, Stream input, Stream output) + { + return (int)GitErrorCode.PassThrough; + } + + /// + /// Smudge the input stream and write to the output stream. + /// + /// The path of the file being filtered + /// The git buf input reader + /// The git buf output writer + /// 0 if successful and to skip and pass through + protected virtual int Smudge(string path, Stream input, Stream output) + { + return (int)GitErrorCode.PassThrough; + } + + /// + /// Determines whether the specified is equal to the current . + /// + /// The to compare with the current . + /// True if the specified is equal to the current ; otherwise, false. + public override bool Equals(object obj) + { + return Equals(obj as Filter); + } + + /// + /// Determines whether the specified is equal to the current . + /// + /// The to compare with the current . + /// True if the specified is equal to the current ; otherwise, false. + public bool Equals(Filter other) + { + return equalityHelper.Equals(this, other); + } + + /// + /// Returns the hash code for this instance. + /// + /// A 32-bit signed integer hash code. + public override int GetHashCode() + { + return equalityHelper.GetHashCode(this); + } + + /// + /// Tests if two are equal. + /// + /// First to compare. + /// Second to compare. + /// True if the two objects are equal; false otherwise. + public static bool operator ==(Filter left, Filter right) + { + return Equals(left, right); + } + + /// + /// Tests if two are different. + /// + /// First to compare. + /// Second to compare. + /// True if the two objects are different; false otherwise. + public static bool operator !=(Filter left, Filter right) + { + return !Equals(left, right); + } + + /// + /// Initialize callback on filter + /// + /// Specified as `filter.initialize`, this is an optional callback invoked + /// before a filter is first used. It will be called once at most. + /// + /// If non-NULL, the filter's `initialize` callback will be invoked right + /// before the first use of the filter, so you can defer expensive + /// initialization operations (in case libgit2 is being used in a way that doesn't need the filter). + /// + int InitializeCallback(IntPtr filterPointer) + { + return Initialize(); + } + } +} diff --git a/LibGit2Sharp/FilterAttributeEntry.cs b/LibGit2Sharp/FilterAttributeEntry.cs new file mode 100644 index 000000000..bd5d434ce --- /dev/null +++ b/LibGit2Sharp/FilterAttributeEntry.cs @@ -0,0 +1,47 @@ +using LibGit2Sharp.Core; + +namespace LibGit2Sharp +{ + /// + /// The definition for a given filter found in the .gitattributes file. + /// The filter definition will result as 'filter=filterName' + /// + /// In the .gitattributes file a filter will be matched to a pathspec like so + /// '*.txt filter=filterName' + /// + public class FilterAttributeEntry + { + private const string AttributeFilterDefinition = "filter="; + + private readonly string filterDefinition; + + /// + /// For testing purposes + /// + protected FilterAttributeEntry() { } + + /// + /// The name of the filter found in a .gitattributes file + /// + /// The name of the filter + public FilterAttributeEntry(string filterName) + { + Ensure.ArgumentNotNullOrEmptyString(filterName, "filterName"); + + if (!filterName.Contains(AttributeFilterDefinition)) + { + filterName = string.Format("{0}{1}", AttributeFilterDefinition, filterName); + } + + this.filterDefinition = filterName; + } + + /// + /// The filter name in the form of 'filter=filterName' + /// + public virtual string FilterDefinition + { + get { return filterDefinition; } + } + } +} diff --git a/LibGit2Sharp/FilterMode.cs b/LibGit2Sharp/FilterMode.cs new file mode 100644 index 000000000..bed687966 --- /dev/null +++ b/LibGit2Sharp/FilterMode.cs @@ -0,0 +1,20 @@ +namespace LibGit2Sharp +{ + /// + /// These values control which direction of change is with which which a filter is being applied. + /// + public enum FilterMode + { + /// + /// Smudge occurs when exporting a file from the Git object database to the working directory. + /// For example, a file would be smudged during a checkout operation. + /// + Smudge = 0, + + /// + /// Clean occurs when importing a file from the working directory to the Git object database. + /// For example, a file would be cleaned when staging a file. + /// + Clean = (1 << 0), + } +} \ No newline at end of file diff --git a/LibGit2Sharp/FilterRegistration.cs b/LibGit2Sharp/FilterRegistration.cs new file mode 100644 index 000000000..9b4ea4e0f --- /dev/null +++ b/LibGit2Sharp/FilterRegistration.cs @@ -0,0 +1,34 @@ +using System; +using System.Runtime.InteropServices; +using LibGit2Sharp.Core; + +namespace LibGit2Sharp +{ + /// + /// An object representing the registration of a Filter type with libgit2 + /// + public sealed class FilterRegistration + { + internal FilterRegistration(Filter filter) + { + Ensure.ArgumentNotNull(filter, "filter"); + Name = filter.Name; + + FilterPointer = Marshal.AllocHGlobal(Marshal.SizeOf(filter.GitFilter)); + Marshal.StructureToPtr(filter.GitFilter, FilterPointer, false); + } + + /// + /// The name of the filter in the libgit2 registry + /// + public string Name { get; private set; } + + internal IntPtr FilterPointer { get; private set; } + + internal void Free() + { + Marshal.FreeHGlobal(FilterPointer); + FilterPointer = IntPtr.Zero; + } + } +} diff --git a/LibGit2Sharp/FilterSource.cs b/LibGit2Sharp/FilterSource.cs new file mode 100644 index 000000000..ee6b2f0b3 --- /dev/null +++ b/LibGit2Sharp/FilterSource.cs @@ -0,0 +1,51 @@ +using System; +using LibGit2Sharp.Core; + +namespace LibGit2Sharp +{ + /// + /// A filter source - describes the direction of filtering and the file being filtered. + /// + public class FilterSource + { + /// + /// Needed for mocking purposes + /// + protected FilterSource() { } + + internal FilterSource(FilePath path, FilterMode mode, GitFilterSource source) + { + SourceMode = mode; + ObjectId = new ObjectId(source.oid); + Path = path.Native; + } + + /// + /// Take an unmanaged pointer and convert it to filter source callback paramater + /// + /// + /// + internal static FilterSource FromNativePtr(IntPtr ptr) + { + var source = ptr.MarshalAs(); + FilePath path = LaxFilePathMarshaler.FromNative(source.path) ?? FilePath.Empty; + FilterMode gitFilterSourceMode = Proxy.git_filter_source_mode(ptr); + return new FilterSource(path, gitFilterSourceMode, source); + } + + /// + /// The filter mode for current file being filtered + /// + public virtual FilterMode SourceMode { get; private set; } + + /// + /// The relative path to the file + /// + public virtual string Path { get; private set; } + + /// + /// The blob id + /// + public virtual ObjectId ObjectId { get; private set; } + } +} diff --git a/LibGit2Sharp/GlobalSettings.cs b/LibGit2Sharp/GlobalSettings.cs index 0aebfc51d..10a3a7aaf 100644 --- a/LibGit2Sharp/GlobalSettings.cs +++ b/LibGit2Sharp/GlobalSettings.cs @@ -169,5 +169,39 @@ internal static string GetAndLockNativeLibraryPath() nativeLibraryPathLocked = true; return nativeLibraryPath; } + + /// + /// Register a filter globally with a default priority of 200 allowing the custom filter + /// to imitate a core Git filter driver. It will be run last on checkout and first on checkin. + /// + public static FilterRegistration RegisterFilter(Filter filter) + { + return RegisterFilter(filter, 200); + } + + /// + /// Register a filter globally with given priority for execution. + /// A filter with the priority of 200 will be run last on checkout and first on checkin. + /// A filter with the priority of 0 will be run first on checkout and last on checkin. + /// + public static FilterRegistration RegisterFilter(Filter filter, int priority) + { + var registration = new FilterRegistration(filter); + + Proxy.git_filter_register(filter.Name, registration.FilterPointer, priority); + + return registration; + } + + /// + /// Remove the filter from the registry, and frees the native heap allocation. + /// + public static void DeregisterFilter(FilterRegistration registration) + { + Ensure.ArgumentNotNull(registration, "registration"); + + Proxy.git_filter_unregister(registration.Name); + registration.Free(); + } } } diff --git a/LibGit2Sharp/LibGit2Sharp.csproj b/LibGit2Sharp/LibGit2Sharp.csproj index abcb70c11..39a245c9a 100644 --- a/LibGit2Sharp/LibGit2Sharp.csproj +++ b/LibGit2Sharp/LibGit2Sharp.csproj @@ -90,6 +90,12 @@ + + + + + + From 4ac3557b637eb27ca4662b99a57a9a0fe04e9a38 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Thu, 23 Apr 2015 09:02:31 -0700 Subject: [PATCH 2/4] Completed filter support via streaming filter API Added complete callback to facilitate long running or transforming filters like Git-LFS. Added repository's working directory to callbacks to support extern process like Git-LFS, where knowledge of a repository's root maybe necissary but unobtainable in another fasion. Removed unused NativeMethods --- LibGit2Sharp.Tests/FilterFixture.cs | 69 +++-- .../FilterSubstitutionCipherFixture.cs | 24 +- LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj | 2 +- .../TestHelpers/SubstitutionCipherFilter.cs | 12 +- LibGit2Sharp/Core/Ensure.cs | 27 ++ LibGit2Sharp/Core/GitFilter.cs | 6 + LibGit2Sharp/Core/GitWriteStream.cs | 22 ++ LibGit2Sharp/Core/NativeMethods.cs | 13 +- LibGit2Sharp/Core/Proxy.cs | 70 ++--- LibGit2Sharp/Filter.cs | 262 +++++++++++++++++- LibGit2Sharp/FilterAttributeEntry.cs | 16 +- LibGit2Sharp/FilterMode.cs | 7 +- LibGit2Sharp/FilterSource.cs | 8 +- LibGit2Sharp/GlobalSettings.cs | 2 +- LibGit2Sharp/LibGit2Sharp.csproj | 1 + 15 files changed, 408 insertions(+), 133 deletions(-) create mode 100644 LibGit2Sharp/Core/GitWriteStream.cs diff --git a/LibGit2Sharp.Tests/FilterFixture.cs b/LibGit2Sharp.Tests/FilterFixture.cs index 776e9c2b2..4f5a2dafa 100644 --- a/LibGit2Sharp.Tests/FilterFixture.cs +++ b/LibGit2Sharp.Tests/FilterFixture.cs @@ -8,9 +8,10 @@ namespace LibGit2Sharp.Tests { public class FilterFixture : BaseFixture { - private const int GitPassThrough = -30; - - readonly Func successCallback = (reader, writer) => 0; + readonly Action successCallback = (reader, writer) => + { + reader.CopyTo(writer); + }; private const string FilterName = "the-filter"; readonly List attributes = new List { new FilterAttributeEntry("test") }; @@ -19,7 +20,7 @@ public class FilterFixture : BaseFixture public void CanRegisterFilterWithSingleAttribute() { var filter = new EmptyFilter(FilterName, attributes); - Assert.Equal( attributes , filter.Attributes); + Assert.Equal(attributes, filter.Attributes); } [Fact] @@ -56,10 +57,9 @@ public void SameFilterIsEqual() public void InitCallbackNotMadeWhenFilterNeverUsed() { bool called = false; - Func initializeCallback = () => + Action initializeCallback = () => { called = true; - return 0; }; var filter = new FakeFilter(FilterName + 11, attributes, @@ -78,10 +78,9 @@ public void InitCallbackNotMadeWhenFilterNeverUsed() public void InitCallbackMadeWhenUsingTheFilter() { bool called = false; - Func initializeCallback = () => + Action initializeCallback = () => { called = true; - return 0; }; var filter = new FakeFilter(FilterName + 12, attributes, @@ -108,10 +107,10 @@ public void WhenStagingFileApplyIsCalledWithCleanForCorrectPath() string repoPath = InitNewRepository(); bool called = false; - Func clean = (reader, writer) => + Action clean = (reader, writer) => { called = true; - return GitPassThrough; + reader.CopyTo(writer); }; var filter = new FakeFilter(FilterName + 15, attributes, clean); @@ -134,7 +133,7 @@ public void CleanFilterWritesOutputToObjectTree() string repoPath = InitNewRepository(); - Func cleanCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; + Action cleanCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; var filter = new FakeFilter(FilterName + 16, attributes, cleanCallback); @@ -154,7 +153,6 @@ public void CleanFilterWritesOutputToObjectTree() GlobalSettings.DeregisterFilter(filterRegistration); } - [Fact] public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory() { @@ -164,7 +162,7 @@ public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory() const string branchName = "branch"; string repoPath = InitNewRepository(); - Func smudgeCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; + Action smudgeCallback = SubstitutionCipherFilter.RotateByThirteenPlaces; var filter = new FakeFilter(FilterName + 17, attributes, null, smudgeCallback); var filterRegistration = GlobalSettings.RegisterFilter(filter); @@ -237,14 +235,14 @@ public EmptyFilter(string name, IEnumerable attributes) class FakeFilter : Filter { - private readonly Func cleanCallback; - private readonly Func smudgeCallback; - private readonly Func initCallback; + private readonly Action cleanCallback; + private readonly Action smudgeCallback; + private readonly Action initCallback; public FakeFilter(string name, IEnumerable attributes, - Func cleanCallback = null, - Func smudgeCallback = null, - Func initCallback = null) + Action cleanCallback = null, + Action smudgeCallback = null, + Action initCallback = null) : base(name, attributes) { this.cleanCallback = cleanCallback; @@ -252,19 +250,40 @@ public FakeFilter(string name, IEnumerable attributes, this.initCallback = initCallback; } - protected override int Clean(string path, Stream input, Stream output) + protected override void Clean(string path, string root, Stream input, Stream output) { - return cleanCallback != null ? cleanCallback(input, output) : base.Clean(path, input, output); + if (cleanCallback == null) + { + base.Clean(path, root, input, output); + } + else + { + cleanCallback(input, output); + } } - protected override int Smudge(string path, Stream input, Stream output) + protected override void Smudge(string path, string root, Stream input, Stream output) { - return smudgeCallback != null ? smudgeCallback(input, output) : base.Smudge(path, input, output); + if (smudgeCallback == null) + { + base.Smudge(path, root, input, output); + } + else + { + smudgeCallback(input, output); + } } - protected override int Initialize() + protected override void Initialize() { - return initCallback != null ? initCallback() : base.Initialize(); + if (initCallback == null) + { + base.Initialize(); + } + else + { + initCallback(); + } } } } diff --git a/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs b/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs index 5f429d001..b2610a574 100644 --- a/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs +++ b/LibGit2Sharp.Tests/FilterSubstitutionCipherFixture.cs @@ -15,7 +15,7 @@ public void SmugdeIsNotCalledForFileWhichDoesNotMatchAnAttributeEntry() const string decodedInput = "This is a substitution cipher"; const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; - var attributes = new List { new FilterAttributeEntry("filter=rot13") }; + var attributes = new List { new FilterAttributeEntry("rot13") }; var filter = new SubstitutionCipherFilter("cipher-filter", attributes); var filterRegistration = GlobalSettings.RegisterFilter(filter); @@ -55,7 +55,7 @@ public void CorrectlyEncodesAndDecodesInput() const string decodedInput = "This is a substitution cipher"; const string encodedInput = "Guvf vf n fhofgvghgvba pvcure"; - var attributes = new List { new FilterAttributeEntry("filter=rot13") }; + var attributes = new List { new FilterAttributeEntry("rot13") }; var filter = new SubstitutionCipherFilter("cipher-filter", attributes); var filterRegistration = GlobalSettings.RegisterFilter(filter); @@ -94,9 +94,9 @@ public void CorrectlyEncodesAndDecodesInput() [InlineData("*.txt", ".txt", 1, 0)] public void WhenStagedFileDoesNotMatchPathSpecFileIsNotFiltered(string pathSpec, string fileExtension, int cleanCount, int smudgeCount) { - const string filterName = "filter=rot13"; + const string filterName = "rot13"; const string decodedInput = "This is a substitution cipher"; - string attributeFileEntry = string.Format("{0} {1}", pathSpec, filterName); + string attributeFileEntry = string.Format("{0} filter={1}", pathSpec, filterName); var filterForAttributes = new List { new FilterAttributeEntry(filterName) }; var filter = new SubstitutionCipherFilter("cipher-filter", filterForAttributes); @@ -122,15 +122,13 @@ public void WhenStagedFileDoesNotMatchPathSpecFileIsNotFiltered(string pathSpec, } [Theory] - [InlineData("filter=rot13", "*.txt filter=rot13", 1)] - [InlineData("filter=rot13", "*.txt filter=fake", 0)] - [InlineData("filter=rot13", "*.bat filter=rot13", 0)] [InlineData("rot13", "*.txt filter=rot13", 1)] [InlineData("rot13", "*.txt filter=fake", 0)] + [InlineData("rot13", "*.bat filter=rot13", 0)] + [InlineData("rot13", "*.txt filter=fake", 0)] [InlineData("fake", "*.txt filter=fake", 1)] - [InlineData("filter=fake", "*.txt filter=fake", 1)] - [InlineData("filter=fake", "*.bat filter=fake", 0)] - [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)] + [InlineData("fake", "*.bat filter=fake", 0)] + [InlineData("rot13", "*.txt filter=rot13 -crlf", 1)] public void CleanIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int cleanCount) { const string decodedInput = "This is a substitution cipher"; @@ -158,12 +156,10 @@ public void CleanIsCalledIfAttributeEntryMatches(string filterAttribute, string } [Theory] - [InlineData("filter=rot13", "*.txt filter=rot13", 1)] - [InlineData("filter=rot13", "*.txt filter=fake", 0)] - [InlineData("filter=rot13", "*.bat filter=rot13", 0)] + [InlineData("rot13", "*.txt filter=rot13", 1)] [InlineData("rot13", "*.txt filter=fake", 0)] - [InlineData("filter=rot13", "*.txt filter=rot13 -crlf", 1)] + [InlineData("rot13", "*.txt filter=rot13 -crlf", 1)] public void SmudgeIsCalledIfAttributeEntryMatches(string filterAttribute, string attributeEntry, int smudgeCount) { const string decodedInput = "This is a substitution cipher"; diff --git a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj index 5b39996a4..539bb19a5 100644 --- a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj +++ b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj @@ -163,4 +163,4 @@ --> - \ No newline at end of file + diff --git a/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs b/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs index ab29b0d77..2cba06d49 100644 --- a/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs +++ b/LibGit2Sharp.Tests/TestHelpers/SubstitutionCipherFilter.cs @@ -13,19 +13,19 @@ public SubstitutionCipherFilter(string name, IEnumerable a { } - protected override int Clean(string path, Stream input, Stream output) + protected override void Clean(string path, string root, Stream input, Stream output) { CleanCalledCount++; - return RotateByThirteenPlaces(input, output); + RotateByThirteenPlaces(input, output); } - protected override int Smudge(string path, Stream input, Stream output) + protected override void Smudge(string path, string root, Stream input, Stream output) { SmudgeCalledCount++; - return RotateByThirteenPlaces(input, output); + RotateByThirteenPlaces(input, output); } - public static int RotateByThirteenPlaces(Stream input, Stream output) + public static void RotateByThirteenPlaces(Stream input, Stream output) { int value; @@ -42,8 +42,6 @@ public static int RotateByThirteenPlaces(Stream input, Stream output) output.WriteByte((byte)value); } - - return 0; } } } diff --git a/LibGit2Sharp/Core/Ensure.cs b/LibGit2Sharp/Core/Ensure.cs index bc9e45506..b051c8505 100644 --- a/LibGit2Sharp/Core/Ensure.cs +++ b/LibGit2Sharp/Core/Ensure.cs @@ -87,6 +87,33 @@ public static void ArgumentDoesNotContainZeroByte(string argumentValue, string a "Zero bytes ('\\0') are not allowed. A zero byte has been found at position {0}.", zeroPos), argumentName); } + /// + /// Checks an argument to ensure it isn't a IntPtr.Zero (aka null). + /// + /// The argument value to check. + /// The name of the argument. + public static void ArgumentNotZeroIntPtr(IntPtr argumentValue, string argumentName) + { + if (argumentValue == IntPtr.Zero) + { + throw new ArgumentNullException(argumentName); + } + } + + /// + /// Checks a pointer argument to ensure it is the expected pointer value. + /// + /// The argument value to check. + /// The expected value. + /// The name of the argument. + public static void ArgumentIsExpectedIntPtr(IntPtr argumentValue, IntPtr expectedValue, string argumentName) + { + if (argumentValue != expectedValue) + { + throw new ArgumentException("Unexpected IntPtr value", argumentName); + } + } + private static readonly Dictionary> GitErrorsToLibGit2SharpExceptions = new Dictionary> diff --git a/LibGit2Sharp/Core/GitFilter.cs b/LibGit2Sharp/Core/GitFilter.cs index 285926bf2..eeb234be5 100644 --- a/LibGit2Sharp/Core/GitFilter.cs +++ b/LibGit2Sharp/Core/GitFilter.cs @@ -24,6 +24,9 @@ internal class GitFilter [MarshalAs(UnmanagedType.FunctionPtr)] public git_filter_apply_fn apply; + [MarshalAs(UnmanagedType.FunctionPtr)] + public git_filter_stream_fn stream; + [MarshalAs(UnmanagedType.FunctionPtr)] public git_filter_cleanup_fn cleanup; @@ -81,6 +84,9 @@ public delegate int git_filter_check_fn( public delegate int git_filter_apply_fn( GitFilter gitFilter, IntPtr payload, IntPtr gitBufTo, IntPtr gitBufFrom, IntPtr filterSource); + public delegate int git_filter_stream_fn( + out IntPtr git_writestream_out, GitFilter self, IntPtr payload, IntPtr filterSource, IntPtr git_writestream_next); + /// /// Callback to clean up after filtering has been applied. Specified as `filter.cleanup`, this is an optional callback invoked /// after the filter has been applied. If the `check` or `apply` callbacks allocated a `payload` diff --git a/LibGit2Sharp/Core/GitWriteStream.cs b/LibGit2Sharp/Core/GitWriteStream.cs new file mode 100644 index 000000000..dc1fd622a --- /dev/null +++ b/LibGit2Sharp/Core/GitWriteStream.cs @@ -0,0 +1,22 @@ +using System; +using System.Runtime.InteropServices; + +namespace LibGit2Sharp.Core +{ + [StructLayout(LayoutKind.Sequential)] + internal class GitWriteStream + { + [MarshalAs(UnmanagedType.FunctionPtr)] + public write_fn write; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public close_fn close; + + [MarshalAs(UnmanagedType.FunctionPtr)] + public free_fn free; + + public delegate int write_fn(IntPtr stream, IntPtr buffer, UIntPtr len); + public delegate int close_fn(IntPtr stream); + public delegate void free_fn(IntPtr stream); + } +} diff --git a/LibGit2Sharp/Core/NativeMethods.cs b/LibGit2Sharp/Core/NativeMethods.cs index b05a1aeb9..b01976ddd 100644 --- a/LibGit2Sharp/Core/NativeMethods.cs +++ b/LibGit2Sharp/Core/NativeMethods.cs @@ -191,12 +191,6 @@ internal static extern int git_branch_remote_name( RepositorySafeHandle repo, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(StrictUtf8Marshaler))] string canonical_branch_name); - [DllImport(libgit2)] - internal static extern int git_buf_grow(IntPtr buffer, UIntPtr targetSize); - - [DllImport(libgit2)] - internal static extern int git_buf_put(IntPtr buffer, IntPtr data, UIntPtr len); - [DllImport(libgit2)] internal static extern int git_remote_rename( ref GitStrArray problems, @@ -523,9 +517,6 @@ internal static extern int git_filter_unregister( [DllImport(libgit2)] internal static extern int git_filter_source_mode(IntPtr source); - [DllImport(libgit2)] - internal static extern void git_filter_free(IntPtr filterSafeHandle); - [DllImport(libgit2)] internal static extern int git_libgit2_features(); @@ -1304,6 +1295,10 @@ internal static extern int git_repository_state( [return: MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(LaxFilePathNoCleanupMarshaler))] internal static extern FilePath git_repository_workdir(RepositorySafeHandle repository); + [DllImport(libgit2)] + [return: MarshalAs(UnmanagedType.CustomMarshaler, MarshalCookie = UniqueId.UniqueIdentifier, MarshalTypeRef = typeof(LaxFilePathNoCleanupMarshaler))] + internal static extern FilePath git_repository_workdir(IntPtr repository); + [DllImport(libgit2)] internal static extern int git_repository_new(out RepositorySafeHandle repo); diff --git a/LibGit2Sharp/Core/Proxy.cs b/LibGit2Sharp/Core/Proxy.cs index 2023eb681..604587b91 100644 --- a/LibGit2Sharp/Core/Proxy.cs +++ b/LibGit2Sharp/Core/Proxy.cs @@ -247,7 +247,7 @@ public static string git_branch_remote_name(RepositorySafeHandle repo, string ca int res = NativeMethods.git_branch_remote_name(buf, repo, canonical_branch_name); if (!shouldThrowIfNotFound && - (res == (int) GitErrorCode.NotFound || res == (int) GitErrorCode.Ambiguous)) + (res == (int)GitErrorCode.NotFound || res == (int)GitErrorCode.Ambiguous)) { return null; } @@ -262,7 +262,7 @@ public static string git_branch_upstream_name(RepositorySafeHandle handle, strin using (var buf = new GitBuf()) { int res = NativeMethods.git_branch_upstream_name(buf, handle, canonicalReferenceName); - if (res == (int) GitErrorCode.NotFound) + if (res == (int)GitErrorCode.NotFound) { return null; } @@ -276,33 +276,6 @@ public static string git_branch_upstream_name(RepositorySafeHandle handle, strin #region git_buf_ - public static void git_buf_grow(IntPtr gitBufPointer, ulong target_size) - { - using (ThreadAffinity()) - { - var res = NativeMethods.git_buf_grow(gitBufPointer, (UIntPtr)target_size); - Ensure.ZeroResult(res); - } - } - - public static void git_buf_put(IntPtr gitBufPointer, byte[] data, int offset, int count) - { - using (ThreadAffinity()) - { - unsafe - { - int res; - - fixed (byte* ptr = data) - { - res = NativeMethods.git_buf_put(gitBufPointer, (IntPtr)ptr, (UIntPtr)count); - } - - Ensure.ZeroResult(res); - } - } - } - public static void git_buf_free(GitBuf buf) { NativeMethods.git_buf_free(buf); @@ -828,34 +801,28 @@ public static int git_diff_num_deltas(DiffSafeHandle diff) public static GitDiffDelta git_diff_get_delta(DiffSafeHandle diff, int idx) { - return NativeMethods.git_diff_get_delta(diff, (UIntPtr) idx).MarshalAs(false); + return NativeMethods.git_diff_get_delta(diff, (UIntPtr)idx).MarshalAs(false); } #endregion #region git_filter_ - public static void git_filter_register(string name, IntPtr filter, int priority) + public static void git_filter_register(string name, FilterRegistration filterRegistration, int priority) { - using (ThreadAffinity()) + int res = NativeMethods.git_filter_register(name, filterRegistration.FilterPointer, priority); + if (res == (int)GitErrorCode.Exists) { - int res = NativeMethods.git_filter_register(name, filter, priority); - if (res == (int)GitErrorCode.Exists) - { - var message = string.Format("A filter with the name '{0}' is already registered", name); - throw new EntryExistsException(message); - } - Ensure.ZeroResult(res); + var message = string.Format("A filter with the name '{0}' is already registered", name); + throw new EntryExistsException(message); } + Ensure.ZeroResult(res); } public static void git_filter_unregister(string name) { - using (ThreadAffinity()) - { - int res = NativeMethods.git_filter_unregister(name); - Ensure.ZeroResult(res); - } + int res = NativeMethods.git_filter_unregister(name); + Ensure.ZeroResult(res); } public static FilterMode git_filter_source_mode(IntPtr filterSource) @@ -864,14 +831,6 @@ public static FilterMode git_filter_source_mode(IntPtr filterSource) return (FilterMode)res; } - public static void git_filter_free(IntPtr gitFilter) - { - using (ThreadAffinity()) - { - NativeMethods.git_filter_free(gitFilter); - } - } - #endregion #region git_graph_ @@ -1817,7 +1776,7 @@ public static bool git_refspec_force(GitRefSpecHandle refSpec) public static TagFetchMode git_remote_autotag(RemoteSafeHandle remote) { - return (TagFetchMode) NativeMethods.git_remote_autotag(remote); + return (TagFetchMode)NativeMethods.git_remote_autotag(remote); } public static RemoteSafeHandle git_remote_create(RepositorySafeHandle repo, string name, string url) @@ -2302,6 +2261,11 @@ public static FilePath git_repository_workdir(RepositorySafeHandle repo) return NativeMethods.git_repository_workdir(repo); } + public static FilePath git_repository_workdir(IntPtr repo) + { + return NativeMethods.git_repository_workdir(repo); + } + public static void git_repository_set_head_detached(RepositorySafeHandle repo, ObjectId commitish) { GitOid oid = commitish.Oid; diff --git a/LibGit2Sharp/Filter.cs b/LibGit2Sharp/Filter.cs index 71947553f..80cb2e2d5 100644 --- a/LibGit2Sharp/Filter.cs +++ b/LibGit2Sharp/Filter.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using LibGit2Sharp.Core; @@ -40,9 +42,16 @@ protected Filter(string name, IEnumerable attributes) { attributes = EncodingMarshaler.FromManaged(Encoding.UTF8, attributesAsString), init = InitializeCallback, + stream = StreamCreateCallback, }; } + private GitWriteStream thisStream; + private GitWriteStream nextStream; + private IntPtr thisPtr; + private IntPtr nextPtr; + private FilterSource filterSource; + /// /// The name that this filter was registered with /// @@ -67,6 +76,19 @@ internal GitFilter GitFilter get { return gitFilter; } } + /// + /// Complete callback on filter + /// + /// This optional callback will be invoked when the upstream filter is + /// closed. Gives the filter a change to perform any final actions or + /// necissary clean up. + /// + /// The path of the file being filtered + /// The path of the working directory for the owning repository + /// Output to the downstream filter or output writer + protected virtual void Complete(string path, string root, Stream output) + { } + /// /// Initialize callback on filter /// @@ -78,33 +100,31 @@ internal GitFilter GitFilter /// initialization operations (in case the library is being used in a way /// that doesn't need the filter. /// - protected virtual int Initialize() - { - return 0; - } + protected virtual void Initialize() + { } /// /// Clean the input stream and write to the output stream. /// /// The path of the file being filtered - /// The git buf input reader - /// The git buf output writer - /// 0 if successful and to skip and pass through - protected virtual int Clean(string path, Stream input, Stream output) + /// The path of the working directory for the owning repository + /// Input from the upstream filter or input reader + /// Output to the downstream filter or output writer + protected virtual void Clean(string path, string root, Stream input, Stream output) { - return (int)GitErrorCode.PassThrough; + input.CopyTo(output); } /// /// Smudge the input stream and write to the output stream. /// /// The path of the file being filtered - /// The git buf input reader - /// The git buf output writer - /// 0 if successful and to skip and pass through - protected virtual int Smudge(string path, Stream input, Stream output) + /// The path of the working directory for the owning repository + /// Input from the upstream filter or input reader + /// Output to the downstream filter or output writer + protected virtual void Smudge(string path, string root, Stream input, Stream output) { - return (int)GitErrorCode.PassThrough; + input.CopyTo(output); } /// @@ -170,7 +190,219 @@ public override int GetHashCode() /// int InitializeCallback(IntPtr filterPointer) { - return Initialize(); + int result = 0; + try + { + Initialize(); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.InitializeCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + return result; + } + + int StreamCreateCallback(out IntPtr git_writestream_out, GitFilter self, IntPtr payload, IntPtr filterSourcePtr, IntPtr git_writestream_next) + { + int result = 0; + + try + { + Ensure.ArgumentNotZeroIntPtr(filterSourcePtr, "filterSourcePtr"); + Ensure.ArgumentNotZeroIntPtr(git_writestream_next, "git_writestream_next"); + + thisStream = new GitWriteStream(); + thisStream.close = StreamCloseCallback; + thisStream.write = StreamWriteCallback; + thisStream.free = StreamFreeCallback; + thisPtr = Marshal.AllocHGlobal(Marshal.SizeOf(thisStream)); + Marshal.StructureToPtr(thisStream, thisPtr, false); + nextPtr = git_writestream_next; + nextStream = new GitWriteStream(); + Marshal.PtrToStructure(nextPtr, nextStream); + filterSource = FilterSource.FromNativePtr(filterSourcePtr); + } + catch (Exception exception) + { + // unexpected failures means memory clean up required + if (thisPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(thisPtr); + thisPtr = IntPtr.Zero; + } + + Log.Write(LogLevel.Error, "Filter.StreamCreateCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + + git_writestream_out = thisPtr; + + return result; + } + + int StreamCloseCallback(IntPtr stream) + { + int result = 0; + + try + { + Ensure.ArgumentNotZeroIntPtr(stream, "stream"); + Ensure.ArgumentIsExpectedIntPtr(stream, thisPtr, "stream"); + + string tempFileName = Path.GetTempFileName(); + // Setup a file system backed write-to stream to work with this gives the runtime + // somewhere to put bits if the amount of data could cause an OOM scenario. + using (FileStream output = File.Open(tempFileName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) + // Setup a septerate read-from stream on the same file system backing + // a seperate stream helps avoid a flush to disk when reading the written content + using (FileStream reader = File.Open(tempFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + Complete(filterSource.Path, filterSource.Root, output); + output.Flush(); + WriteToNextFilter(reader); + } + // clean up after outselves + File.Delete(tempFileName); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.StreamCloseCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + + result = nextStream.close(nextPtr); + + return result; + } + + void StreamFreeCallback(IntPtr stream) + { + try + { + Ensure.ArgumentNotZeroIntPtr(stream, "stream"); + Ensure.ArgumentIsExpectedIntPtr(stream, thisPtr, "stream"); + + Marshal.FreeHGlobal(thisPtr); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.StreamFreeCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + } + } + + unsafe int StreamWriteCallback(IntPtr stream, IntPtr buffer, UIntPtr len) + { + int result = 0; + + try + { + Ensure.ArgumentNotZeroIntPtr(stream, "stream"); + Ensure.ArgumentNotZeroIntPtr(buffer, "buffer"); + Ensure.ArgumentIsExpectedIntPtr(stream, thisPtr, "stream"); + + string tempFileName = Path.GetTempFileName(); + using (UnmanagedMemoryStream input = new UnmanagedMemoryStream((byte*)buffer.ToPointer(), (long)len)) + // Setup a file system backed write-to stream to work with this gives the runtime + // somewhere to put bits if the amount of data could cause an OOM scenario. + using (FileStream output = File.Open(tempFileName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) + // Setup a septerate read-from stream on the same file system backing + // a seperate stream helps avoid a flush to disk when reading the written content + using (FileStream reader = File.Open(tempFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + switch (filterSource.SourceMode) + { + case FilterMode.Clean: + try + { + Clean(filterSource.Path, filterSource.Root, input, output); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.StreamWriteCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + break; + + case FilterMode.Smudge: + try + { + Smudge(filterSource.Path, filterSource.Root, input, output); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.StreamWriteCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + break; + default: + Proxy.giterr_set_str(GitErrorCategory.Filter, "Unexpected filter mode."); + return (int)GitErrorCode.Ambiguous; + } + + if (result == (int)GitErrorCode.Ok) + { + // have to flush the write-to stream to enable the read stream to get access to the bits + output.Flush(); + result = WriteToNextFilter(reader); + } + } + + // clean up after outselves + File.Delete(tempFileName); + } + catch (Exception exception) + { + Log.Write(LogLevel.Error, "Filter.StreamWriteCallback exception"); + Log.Write(LogLevel.Error, exception.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Filter, exception); + result = (int)GitErrorCode.Error; + } + + return result; + } + + private unsafe int WriteToNextFilter(Stream output) + { + // 64K is optimal buffer size per https://technet.microsoft.com/en-us/library/cc938632.aspx + const int BufferSize = 64 * 1024; + + Debug.Assert(output != null, "output parameter is null"); + Debug.Assert(output.CanRead, "output.CanRead parameter equals false"); + + int result = 0; + byte[] bytes = new byte[BufferSize]; + IntPtr bytesPtr = Marshal.AllocHGlobal(BufferSize); + try + { + int read = 0; + while ((read = output.Read(bytes, 0, bytes.Length)) > 0) + { + Marshal.Copy(bytes, 0, bytesPtr, read); + if ((result = nextStream.write(nextPtr, bytesPtr, (UIntPtr)read)) != (int)GitErrorCode.Ok) + { + Proxy.giterr_set_str(GitErrorCategory.Filter, "Filter write to next stream failed"); + break; + } + } + } + finally + { + Marshal.FreeHGlobal(bytesPtr); + } + + return result; } } } diff --git a/LibGit2Sharp/FilterAttributeEntry.cs b/LibGit2Sharp/FilterAttributeEntry.cs index bd5d434ce..117523d3e 100644 --- a/LibGit2Sharp/FilterAttributeEntry.cs +++ b/LibGit2Sharp/FilterAttributeEntry.cs @@ -1,3 +1,4 @@ +using System; using LibGit2Sharp.Core; namespace LibGit2Sharp @@ -21,18 +22,23 @@ public class FilterAttributeEntry protected FilterAttributeEntry() { } /// - /// The name of the filter found in a .gitattributes file + /// The name of the filter found in a .gitattributes file. /// - /// The name of the filter + /// The name of the filter as found in the .gitattributes file without the "filter=" prefix + /// + /// "filter=" will be prepended to the filterDefinition, therefore the "filter=" portion of the filter + /// name shouldbe omitted on declaration. Inclusion of the "filter=" prefix will cause the FilterDefinition to + /// fail to match the .gitattributes entry and thefore no be invoked correctly. + /// public FilterAttributeEntry(string filterName) { Ensure.ArgumentNotNullOrEmptyString(filterName, "filterName"); - - if (!filterName.Contains(AttributeFilterDefinition)) + if (filterName.StartsWith("filter=", StringComparison.OrdinalIgnoreCase)) { - filterName = string.Format("{0}{1}", AttributeFilterDefinition, filterName); + throw new ArgumentException("The filterName parameter should not begin with \"filter=\"", filterName); } + filterName = AttributeFilterDefinition + filterName; this.filterDefinition = filterName; } diff --git a/LibGit2Sharp/FilterMode.cs b/LibGit2Sharp/FilterMode.cs index bed687966..31a9546c1 100644 --- a/LibGit2Sharp/FilterMode.cs +++ b/LibGit2Sharp/FilterMode.cs @@ -3,6 +3,9 @@ namespace LibGit2Sharp /// /// These values control which direction of change is with which which a filter is being applied. /// + /// + /// These enum values must be identical to the values in Libgit2 filter_mode_t found in filter.h + /// public enum FilterMode { /// @@ -15,6 +18,6 @@ public enum FilterMode /// Clean occurs when importing a file from the working directory to the Git object database. /// For example, a file would be cleaned when staging a file. /// - Clean = (1 << 0), + Clean = 1, } -} \ No newline at end of file +} diff --git a/LibGit2Sharp/FilterSource.cs b/LibGit2Sharp/FilterSource.cs index ee6b2f0b3..0843e6221 100644 --- a/LibGit2Sharp/FilterSource.cs +++ b/LibGit2Sharp/FilterSource.cs @@ -11,13 +11,14 @@ public class FilterSource /// /// Needed for mocking purposes /// - protected FilterSource() { } + protected FilterSource() { } internal FilterSource(FilePath path, FilterMode mode, GitFilterSource source) { SourceMode = mode; ObjectId = new ObjectId(source.oid); Path = path.Native; + Root = Proxy.git_repository_workdir(source.repository).Native; } /// @@ -47,5 +48,10 @@ internal static FilterSource FromNativePtr(IntPtr ptr) /// The blob id /// public virtual ObjectId ObjectId { get; private set; } + + /// + /// The working directory + /// + public virtual string Root { get; private set; } } } diff --git a/LibGit2Sharp/GlobalSettings.cs b/LibGit2Sharp/GlobalSettings.cs index 10a3a7aaf..d3eca3aea 100644 --- a/LibGit2Sharp/GlobalSettings.cs +++ b/LibGit2Sharp/GlobalSettings.cs @@ -188,7 +188,7 @@ public static FilterRegistration RegisterFilter(Filter filter, int priority) { var registration = new FilterRegistration(filter); - Proxy.git_filter_register(filter.Name, registration.FilterPointer, priority); + Proxy.git_filter_register(filter.Name, registration, priority); return registration; } diff --git a/LibGit2Sharp/LibGit2Sharp.csproj b/LibGit2Sharp/LibGit2Sharp.csproj index 39a245c9a..97a58aaa1 100644 --- a/LibGit2Sharp/LibGit2Sharp.csproj +++ b/LibGit2Sharp/LibGit2Sharp.csproj @@ -74,6 +74,7 @@ + From def0c7bb937e3ee39961ac0309daa46a0179c8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Thu, 28 May 2015 17:20:30 +0200 Subject: [PATCH 3/4] PoC for writing directly into the next stream The filters are run in-line with the rest of the system, and libgit2 is set up to have its last filter/writestream write to the filesystem. Instead of using an intermediate buffer/file/pipe, we can make the writes from the user's Smude() or Clean() filter into their output stream propagate immediately into the next filter, leading up the chain to the filesystem, removing any intermediate buffering other than that which is necessary for the filters themselves to work. In the general case, we only have the one clean or smudge filter, which means that even in the case of out-of-band storage for large files, where a small blob can cause very large outputs, we would still write directly to the filesystem. by J Wyman (@whoisj): Buffered writes to next filter to minimize managed to native marshaling for performance optimizations by J Wyman (@whoisj): Split `WriteStream` into its own file and cleaned it up (checks, style, etc) --- LibGit2Sharp/Core/WriteStream.cs | 64 ++++++++++++++++++++++ LibGit2Sharp/Filter.cs | 92 ++++---------------------------- LibGit2Sharp/LibGit2Sharp.csproj | 1 + 3 files changed, 76 insertions(+), 81 deletions(-) create mode 100644 LibGit2Sharp/Core/WriteStream.cs diff --git a/LibGit2Sharp/Core/WriteStream.cs b/LibGit2Sharp/Core/WriteStream.cs new file mode 100644 index 000000000..37db8af8c --- /dev/null +++ b/LibGit2Sharp/Core/WriteStream.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; + +namespace LibGit2Sharp.Core +{ + class WriteStream : Stream + { + readonly GitWriteStream nextStream; + readonly IntPtr nextPtr; + + public WriteStream(GitWriteStream nextStream, IntPtr nextPtr) + { + this.nextStream = nextStream; + this.nextPtr = nextPtr; + } + + public override bool CanWrite { get { return true; } } + + public override bool CanRead { get { return false; } } + + public override bool CanSeek { get { return false; } } + + public override long Position + { + get { throw new NotImplementedException(); } + set { throw new InvalidOperationException(); } + } + + public override long Length { get { throw new InvalidOperationException(); } } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new InvalidOperationException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new InvalidOperationException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new InvalidOperationException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + unsafe + { + fixed (byte* bufferPtr = &buffer[offset]) + { + if (nextStream.write(nextPtr, (IntPtr)bufferPtr, (UIntPtr)count) < 0) + { + throw new LibGit2SharpException("failed to write to next buffer"); + } + } + } + } + } +} diff --git a/LibGit2Sharp/Filter.cs b/LibGit2Sharp/Filter.cs index 80cb2e2d5..56cee570a 100644 --- a/LibGit2Sharp/Filter.cs +++ b/LibGit2Sharp/Filter.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -17,6 +16,8 @@ public abstract class Filter : IEquatable { private static readonly LambdaEqualityHelper equalityHelper = new LambdaEqualityHelper(x => x.Name, x => x.Attributes); + // 64K is optimal buffer size per https://technet.microsoft.com/en-us/library/cc938632.aspx + private const int BufferSize = 64 * 1024; private readonly string name; private readonly IEnumerable attributes; @@ -51,6 +52,7 @@ protected Filter(string name, IEnumerable attributes) private IntPtr thisPtr; private IntPtr nextPtr; private FilterSource filterSource; + private Stream output; /// /// The name that this filter was registered with @@ -80,7 +82,7 @@ internal GitFilter GitFilter /// Complete callback on filter /// /// This optional callback will be invoked when the upstream filter is - /// closed. Gives the filter a change to perform any final actions or + /// closed. Gives the filter a chance to perform any final actions or /// necissary clean up. /// /// The path of the file being filtered @@ -224,6 +226,7 @@ int StreamCreateCallback(out IntPtr git_writestream_out, GitFilter self, IntPtr nextStream = new GitWriteStream(); Marshal.PtrToStructure(nextPtr, nextStream); filterSource = FilterSource.FromNativePtr(filterSourcePtr); + output = new WriteStream(nextStream, nextPtr); } catch (Exception exception) { @@ -254,20 +257,10 @@ int StreamCloseCallback(IntPtr stream) Ensure.ArgumentNotZeroIntPtr(stream, "stream"); Ensure.ArgumentIsExpectedIntPtr(stream, thisPtr, "stream"); - string tempFileName = Path.GetTempFileName(); - // Setup a file system backed write-to stream to work with this gives the runtime - // somewhere to put bits if the amount of data could cause an OOM scenario. - using (FileStream output = File.Open(tempFileName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) - // Setup a septerate read-from stream on the same file system backing - // a seperate stream helps avoid a flush to disk when reading the written content - using (FileStream reader = File.Open(tempFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (BufferedStream outputBuffer = new BufferedStream(output, BufferSize)) { - Complete(filterSource.Path, filterSource.Root, output); - output.Flush(); - WriteToNextFilter(reader); + Complete(filterSource.Path, filterSource.Root, outputBuffer); } - // clean up after outselves - File.Delete(tempFileName); } catch (Exception exception) { @@ -310,53 +303,22 @@ unsafe int StreamWriteCallback(IntPtr stream, IntPtr buffer, UIntPtr len) string tempFileName = Path.GetTempFileName(); using (UnmanagedMemoryStream input = new UnmanagedMemoryStream((byte*)buffer.ToPointer(), (long)len)) - // Setup a file system backed write-to stream to work with this gives the runtime - // somewhere to put bits if the amount of data could cause an OOM scenario. - using (FileStream output = File.Open(tempFileName, FileMode.Open, FileAccess.Write, FileShare.ReadWrite)) - // Setup a septerate read-from stream on the same file system backing - // a seperate stream helps avoid a flush to disk when reading the written content - using (FileStream reader = File.Open(tempFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (BufferedStream outputBuffer = new BufferedStream(output, BufferSize)) { switch (filterSource.SourceMode) { case FilterMode.Clean: - try - { - Clean(filterSource.Path, filterSource.Root, input, output); - } - catch (Exception exception) - { - Log.Write(LogLevel.Error, "Filter.StreamWriteCallback exception"); - Log.Write(LogLevel.Error, exception.ToString()); - Proxy.giterr_set_str(GitErrorCategory.Filter, exception); - result = (int)GitErrorCode.Error; - } + Clean(filterSource.Path, filterSource.Root, input, outputBuffer); break; case FilterMode.Smudge: - try - { - Smudge(filterSource.Path, filterSource.Root, input, output); - } - catch (Exception exception) - { - Log.Write(LogLevel.Error, "Filter.StreamWriteCallback exception"); - Log.Write(LogLevel.Error, exception.ToString()); - Proxy.giterr_set_str(GitErrorCategory.Filter, exception); - result = (int)GitErrorCode.Error; - } + Smudge(filterSource.Path, filterSource.Root, input, outputBuffer); break; + default: Proxy.giterr_set_str(GitErrorCategory.Filter, "Unexpected filter mode."); return (int)GitErrorCode.Ambiguous; } - - if (result == (int)GitErrorCode.Ok) - { - // have to flush the write-to stream to enable the read stream to get access to the bits - output.Flush(); - result = WriteToNextFilter(reader); - } } // clean up after outselves @@ -372,37 +334,5 @@ unsafe int StreamWriteCallback(IntPtr stream, IntPtr buffer, UIntPtr len) return result; } - - private unsafe int WriteToNextFilter(Stream output) - { - // 64K is optimal buffer size per https://technet.microsoft.com/en-us/library/cc938632.aspx - const int BufferSize = 64 * 1024; - - Debug.Assert(output != null, "output parameter is null"); - Debug.Assert(output.CanRead, "output.CanRead parameter equals false"); - - int result = 0; - byte[] bytes = new byte[BufferSize]; - IntPtr bytesPtr = Marshal.AllocHGlobal(BufferSize); - try - { - int read = 0; - while ((read = output.Read(bytes, 0, bytes.Length)) > 0) - { - Marshal.Copy(bytes, 0, bytesPtr, read); - if ((result = nextStream.write(nextPtr, bytesPtr, (UIntPtr)read)) != (int)GitErrorCode.Ok) - { - Proxy.giterr_set_str(GitErrorCategory.Filter, "Filter write to next stream failed"); - break; - } - } - } - finally - { - Marshal.FreeHGlobal(bytesPtr); - } - - return result; - } } } diff --git a/LibGit2Sharp/LibGit2Sharp.csproj b/LibGit2Sharp/LibGit2Sharp.csproj index 97a58aaa1..721530932 100644 --- a/LibGit2Sharp/LibGit2Sharp.csproj +++ b/LibGit2Sharp/LibGit2Sharp.csproj @@ -75,6 +75,7 @@ + From 71dc2c1a74a7b298c07a07a27c0751b763c418e4 Mon Sep 17 00:00:00 2001 From: J Wyman Date: Fri, 29 May 2015 08:56:00 -0700 Subject: [PATCH 4/4] Added LFS like test to FilterFixture --- LibGit2Sharp.Tests/FilterFixture.cs | 107 ++++++++++++++++++ LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj | 3 + .../TestHelpers/FileExportFilter.cs | 83 ++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 LibGit2Sharp.Tests/TestHelpers/FileExportFilter.cs diff --git a/LibGit2Sharp.Tests/FilterFixture.cs b/LibGit2Sharp.Tests/FilterFixture.cs index 4f5a2dafa..1de81595d 100644 --- a/LibGit2Sharp.Tests/FilterFixture.cs +++ b/LibGit2Sharp.Tests/FilterFixture.cs @@ -176,6 +176,113 @@ public void WhenCheckingOutAFileFileSmudgeWritesCorrectFileToWorkingDirectory() GlobalSettings.DeregisterFilter(filterRegistration); } + [Fact] + public void CanFilterLargeFiles() + { + const int ContentLength = 128 * 1024 * 1024; + const char ContentValue = 'x'; + + char[] content = (new string(ContentValue, 1024)).ToCharArray(); + + string repoPath = InitNewRepository(); + + var filter = new FileExportFilter("exportFilter", attributes); + var filterRegistration = GlobalSettings.RegisterFilter(filter); + + string filePath = Path.Combine(Directory.GetParent(repoPath).Parent.FullName, Guid.NewGuid().ToString() + ".blob"); + FileInfo contentFile = new FileInfo(filePath); + using (var writer = new StreamWriter(contentFile.OpenWrite()) { AutoFlush = true }) + { + for (int i = 0; i < ContentLength / content.Length; i++) + { + writer.Write(content); + } + } + + string attributesPath = Path.Combine(Directory.GetParent(repoPath).Parent.FullName, ".gitattributes"); + FileInfo attributesFile = new FileInfo(attributesPath); + + string configPath = CreateConfigurationWithDummyUser(Constants.Signature); + var repositoryOptions = new RepositoryOptions { GlobalConfigurationLocation = configPath }; + + using (Repository repo = new Repository(repoPath, repositoryOptions)) + { + File.WriteAllText(attributesPath, "*.blob filter=test"); + repo.Stage(attributesFile.Name); + repo.Stage(contentFile.Name); + repo.Commit("test"); + contentFile.Delete(); + repo.Checkout("HEAD", new CheckoutOptions() { CheckoutModifiers = CheckoutModifiers.Force }); + } + + contentFile = new FileInfo(filePath); + Assert.True(contentFile.Exists, "Contents not restored correctly by forced checkout."); + using (StreamReader reader = contentFile.OpenText()) + { + int totalRead = 0; + char[] block = new char[1024]; + int read; + while ((read = reader.Read(block, 0, block.Length)) > 0) + { + Assert.True(CharArrayAreEqual(block, content, read)); + totalRead += read; + } + + Assert.Equal(ContentLength, totalRead); + } + + contentFile.Delete(); + + GlobalSettings.DeregisterFilter(filterRegistration); + } + + private unsafe bool CharArrayAreEqual(char[] array1, char[] array2, int count) + { + if (Object.ReferenceEquals(array1, array2)) + { + return true; + } + if (Object.ReferenceEquals(array1, null) || Object.ReferenceEquals(null, array2)) + { + return false; + } + if (array1.Length < count || array2.Length < count) + { + return false; + } + + int len = count * sizeof(char); + int cnt = len / sizeof(long); + + fixed (char* c1 = array1, c2 = array2) + { + long* p1 = (long*)c1, + p2 = (long*)c2; + + for (int i = 0; i < cnt; i++) + { + if (p1[i] != p2[i]) + { + return false; + } + } + + byte* b1 = (byte*)c1, + b2 = (byte*)c2; + + for (int i = len * sizeof(long); i < len; i++) + { + if (b1[i] != b2[i]) + { + return false; + } + } + } + + return true; + } + + private FileInfo CheckoutFileForSmudge(string repoPath, string branchName, string content) { FileInfo expectedPath; diff --git a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj index 539bb19a5..aca844148 100644 --- a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj +++ b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj @@ -23,6 +23,7 @@ TRACE;DEBUG;NET40 prompt 4 + true pdbonly @@ -31,6 +32,7 @@ TRACE prompt 4 + true @@ -118,6 +120,7 @@ + diff --git a/LibGit2Sharp.Tests/TestHelpers/FileExportFilter.cs b/LibGit2Sharp.Tests/TestHelpers/FileExportFilter.cs new file mode 100644 index 000000000..3036be414 --- /dev/null +++ b/LibGit2Sharp.Tests/TestHelpers/FileExportFilter.cs @@ -0,0 +1,83 @@ +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace LibGit2Sharp.Tests.TestHelpers +{ + class FileExportFilter : Filter + { + public int CleanCalledCount = 0; + public int CompleteCalledCount = 0; + public int SmudgeCalledCount = 0; + public readonly HashSet FilesFiltered; + + private bool clean; + + public FileExportFilter(string name, IEnumerable attributes) + : base(name, attributes) + { + FilesFiltered = new HashSet(); + } + + protected override void Clean(string path, string root, Stream input, Stream output) + { + CleanCalledCount++; + + string filename = Path.GetFileName(path); + string cachePath = Path.Combine(root, ".git", filename); + + using (var file = File.Exists(cachePath) ? File.Open(cachePath, FileMode.Append, FileAccess.Write, FileShare.None) : File.Create(cachePath)) + { + input.CopyTo(file); + } + + clean = true; + } + + protected override void Complete(string path, string root, Stream output) + { + CompleteCalledCount++; + + string filename = Path.GetFileName(path); + string cachePath = Path.Combine(root, ".git", filename); + + if (clean) + { + byte[] bytes = Encoding.UTF8.GetBytes(path); + output.Write(bytes, 0, bytes.Length); + FilesFiltered.Add(path); + } + else + { + if (File.Exists(cachePath)) + { + using (var file = File.Open(cachePath, FileMode.OpenOrCreate, FileAccess.Read, FileShare.None)) + { + file.CopyTo(output); + } + } + } + } + + protected override void Smudge(string path, string root, Stream input, Stream output) + { + SmudgeCalledCount++; + + string filename = Path.GetFileName(path); + StringBuilder text = new StringBuilder(); + + byte[] buffer = new byte[64 * 1024]; + int read; + while ((read = input.Read(buffer, 0, buffer.Length)) > 0) + { + string decoded = Encoding.UTF8.GetString(buffer, 0, read); + text.Append(decoded); + } + + if (!FilesFiltered.Contains(text.ToString())) + throw new FileNotFoundException(); + + clean = false; + } + } +}