From 185cefb1c5617f74d45668aabb25ebaa091825bf Mon Sep 17 00:00:00 2001 From: Brendon Page Date: Sun, 21 Aug 2022 22:20:39 +0400 Subject: [PATCH 1/4] Added support for version >= 8.0.0 * GenerateCertificatesTask now generates a certificate authority when version 8 detected * ClusterComposeTask ExecuteBinary now also ignores lines containing "using ES_JAVA_HOME" --- .../EphemeralFileSystem.cs | 1 + .../XPack/GenerateCertificatesTask.cs | 65 ++++++++++++++----- .../Tasks/IClusterComposeTask.cs | 4 +- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/Elastic.Elasticsearch.Ephemeral/EphemeralFileSystem.cs b/src/Elastic.Elasticsearch.Ephemeral/EphemeralFileSystem.cs index 6551fee..15acaa7 100644 --- a/src/Elastic.Elasticsearch.Ephemeral/EphemeralFileSystem.cs +++ b/src/Elastic.Elasticsearch.Ephemeral/EphemeralFileSystem.cs @@ -32,6 +32,7 @@ public EphemeralFileSystem(ElasticVersion version, string clusterName) : base(ve public string CertificatesPath => Path.Combine(ConfigPath, CertificateFolderName); public string CaCertificate => Path.Combine(CertificatesPath, "ca", "ca") + ".crt"; + public string CaPrivateKey => Path.Combine(CertificatesPath, "ca", "ca") + ".key"; public string NodePrivateKey => Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".key"; diff --git a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs index da7a0c2..e89e1fe 100644 --- a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs +++ b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs @@ -2,6 +2,7 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System; using System.IO; using System.IO.Compression; using System.Linq; @@ -80,7 +81,12 @@ private static void GenerateCertificates(IEphemeralCluster GenerateCaCertificate(config, zipLocation, writer) + ); + NewOrCachedCertificates(cluster, name, path, writer, + zipLocation => GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer) + ); } private static void GenerateUnusedCertificates(IEphemeralCluster cluster, @@ -89,11 +95,16 @@ private static void GenerateUnusedCertificates(IEphemeralCluster GenerateCaCertificate(config, zipLocation, writer) + ); + NewOrCachedCertificates(cluster, name, path, writer, + zipLocation => GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer) + ); } private static void NewOrCachedCertificates(IEphemeralCluster cluster, - string name, string path, string silentModeConfigFile, IConsoleLineHandler writer) + string name, string path, IConsoleLineHandler writer, Action generateCertificateAction) { var config = cluster.ClusterConfiguration; var cachedEsHomeFolder = Path.Combine(config.FileSystem.LocalFolder, cluster.GetCacheFolderName()); @@ -110,7 +121,7 @@ private static void NewOrCachedCertificates(IEphemeralCluster= "6.3.0" ? Path.Combine(config.FileSystem.ConfigPath, name) + ".zip" : Path.Combine(config.FileSystem.ConfigPath, "x-pack", name) + ".zip"; - GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer); + generateCertificateAction(zipLocation); if (!File.Exists(zipLocationCache)) { @@ -133,17 +144,21 @@ private static void GenerateCertificate(EphemeralClusterConfiguration config, st : Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix : Path.Combine(fs.ElasticsearchHome, "bin", "x-pack", "certgen") + BinarySuffix; - - if (!Directory.Exists(path)) - { - if (config.Version < "7.0.0") - ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", - "-in", silentModeConfigFile, "-out", @out); - else - ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", - "cert", - "-in", silentModeConfigFile, "-out", @out); - } + // TODO figure out what to do about the original check for the existence of 'path', + // I suspect it was done so that the 'NoCleanupAfterNodeStopped' option can be + // to launch a cluster that was setup in the past. Perhaps I can move this + // check to be higher up in the call chain. + if (config.Version < "7.0.0") + ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", + "-in", silentModeConfigFile, "-out", @out); + else if (config.Version < "8.0.0") + ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", + "cert", + "--in", silentModeConfigFile, "--out", @out); + else + ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", + "cert", "--pem", + "--in", silentModeConfigFile, "--out", @out, "--ca-cert", fs.CaCertificate, "--ca-key", fs.CaPrivateKey); var badLocation = Path.Combine(config.FileSystem.ElasticsearchHome, "config", "x-pack", @out); //not necessary anymore now that we patch .in.bat i think @@ -154,14 +169,32 @@ private static void GenerateCertificate(EphemeralClusterConfiguration config, st } } + private static void GenerateCaCertificate(EphemeralClusterConfiguration config, + string zipLocation, IConsoleLineHandler writer) + { + if (config.Version < "8.0.0") return; + + var @out = zipLocation; + var fs = config.FileSystem; + var binary = Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix; + + ExecuteBinary(config, writer, binary, "generating CA certificate for this session", + "ca", "--pem", "--out", @out); + } + private static void UnpackCertificatesZip(string zipLocation, string outFolder, IConsoleLineHandler writer) { - if (Directory.Exists(outFolder)) return; + // TODO figure out what to do about the original check for the existence of 'path', + // I suspect it was done so that the 'NoCleanupAfterNodeStopped' option can be + // to launch a cluster that was setup in the past. Perhaps I can move this + // check to be higher up in the call chain. writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} unzipping certificates to {outFolder}"); Directory.CreateDirectory(outFolder); + ZipFile.ExtractToDirectory(zipLocation, outFolder); + } } } diff --git a/src/Elastic.Elasticsearch.Ephemeral/Tasks/IClusterComposeTask.cs b/src/Elastic.Elasticsearch.Ephemeral/Tasks/IClusterComposeTask.cs index d13e990..e741ebf 100644 --- a/src/Elastic.Elasticsearch.Ephemeral/Tasks/IClusterComposeTask.cs +++ b/src/Elastic.Elasticsearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -185,8 +185,8 @@ private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, errorOut = errorOut.Where(e => !e.Line.Contains("No log4j2 configuration file found")).ToList(); if (errorOut.Any(e => - !string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated")) && - !binary.Contains("plugin") && !binary.Contains("cert")) + !string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated") && !e.Line.Contains("using ES_JAVA_HOME")) && + !binary.Contains("plugin") && !binary.Contains("cert") ) throw new Exception( $"Received error out with exitCode ({result.ExitCode}) while executing {description}: {command}"); From 97aa93a95dbed9693f82888d891e716558024a12 Mon Sep 17 00:00:00 2001 From: Brendon Page Date: Mon, 22 Aug 2022 14:33:21 +0400 Subject: [PATCH 2/4] Skip generating CA for versions less than 8 * Added check back in GenerateCertificatesTask to skip certificate generation if certificate path already exists --- .../XPack/GenerateCertificatesTask.cs | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs index e89e1fe..b8d5529 100644 --- a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs +++ b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs @@ -40,6 +40,12 @@ public override void Run(IEphemeralCluster cluste if (!cluster.ClusterConfiguration.EnableSsl) return; var config = cluster.ClusterConfiguration; + + if (Directory.Exists(config.FileSystem.CertificatesPath)) + { + cluster.Writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} Skipping certificate generation as ${{{config.FileSystem.CertificatesPath}}} already exists"); + return; + } var fileSystem = cluster.FileSystem; //due to a bug in certgen this file needs to live in two places @@ -82,8 +88,8 @@ private static void GenerateCertificates(IEphemeralCluster GenerateCaCertificate(config, zipLocation, writer) - ); + zipLocation => GenerateCaCertificate(config, zipLocation, writer), + "8.0.0"); NewOrCachedCertificates(cluster, name, path, writer, zipLocation => GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer) ); @@ -96,20 +102,22 @@ private static void GenerateUnusedCertificates(IEphemeralCluster GenerateCaCertificate(config, zipLocation, writer) - ); + zipLocation => GenerateCaCertificate(config, zipLocation, writer), + "8.0.0"); NewOrCachedCertificates(cluster, name, path, writer, zipLocation => GenerateCertificate(config, name, path, zipLocation, silentModeConfigFile, writer) ); } private static void NewOrCachedCertificates(IEphemeralCluster cluster, - string name, string path, IConsoleLineHandler writer, Action generateCertificateAction) + string name, string path, IConsoleLineHandler writer, Action generateCertificateAction, string minVersion = null) { var config = cluster.ClusterConfiguration; var cachedEsHomeFolder = Path.Combine(config.FileSystem.LocalFolder, cluster.GetCacheFolderName()); var zipLocationCache = Path.Combine(cachedEsHomeFolder, name) + ".zip"; + if (minVersion != null && config.Version < minVersion) return; + if (File.Exists(zipLocationCache)) { writer.WriteDiagnostic( @@ -144,10 +152,6 @@ private static void GenerateCertificate(EphemeralClusterConfiguration config, st : Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix : Path.Combine(fs.ElasticsearchHome, "bin", "x-pack", "certgen") + BinarySuffix; - // TODO figure out what to do about the original check for the existence of 'path', - // I suspect it was done so that the 'NoCleanupAfterNodeStopped' option can be - // to launch a cluster that was setup in the past. Perhaps I can move this - // check to be higher up in the call chain. if (config.Version < "7.0.0") ExecuteBinary(config, writer, binary, "generating ssl certificates for this session", "-in", silentModeConfigFile, "-out", @out); @@ -185,11 +189,6 @@ private static void GenerateCaCertificate(EphemeralClusterConfiguration config, private static void UnpackCertificatesZip(string zipLocation, string outFolder, IConsoleLineHandler writer) { - // TODO figure out what to do about the original check for the existence of 'path', - // I suspect it was done so that the 'NoCleanupAfterNodeStopped' option can be - // to launch a cluster that was setup in the past. Perhaps I can move this - // check to be higher up in the call chain. - writer.WriteDiagnostic($"{{{nameof(GenerateCertificatesTask)}}} unzipping certificates to {outFolder}"); Directory.CreateDirectory(outFolder); From 58c6a525a6867f9e982baa765b443f74d64089c2 Mon Sep 17 00:00:00 2001 From: Brendon Page Date: Mon, 22 Aug 2022 17:04:41 +0400 Subject: [PATCH 3/4] Removed unnecessary variables from CA generation. --- .../BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs index b8d5529..c8a6b0a 100644 --- a/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs +++ b/src/Elastic.Elasticsearch.Ephemeral/Tasks/BeforeStartNodeTasks/XPack/GenerateCertificatesTask.cs @@ -178,12 +178,10 @@ private static void GenerateCaCertificate(EphemeralClusterConfiguration config, { if (config.Version < "8.0.0") return; - var @out = zipLocation; - var fs = config.FileSystem; - var binary = Path.Combine(fs.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix; + var binary = Path.Combine(config.FileSystem.ElasticsearchHome, "bin", "elasticsearch-certutil") + BinarySuffix; ExecuteBinary(config, writer, binary, "generating CA certificate for this session", - "ca", "--pem", "--out", @out); + "ca", "--pem", "--out", zipLocation); } From 00bd24ca296f8299a866c329d0931b966514dd72 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 30 Jun 2023 14:43:20 +0200 Subject: [PATCH 4/4] update example to validate certificate generation --- examples/Elastic.Ephemeral.Example/Program.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/Elastic.Ephemeral.Example/Program.cs b/examples/Elastic.Ephemeral.Example/Program.cs index f4cdb1c..5d03b9b 100644 --- a/examples/Elastic.Ephemeral.Example/Program.cs +++ b/examples/Elastic.Ephemeral.Example/Program.cs @@ -3,7 +3,17 @@ // See the LICENSE file in the project root for more information using Elastic.Elasticsearch.Ephemeral; +using static Elastic.Elasticsearch.Ephemeral.ClusterFeatures; -var config = new EphemeralClusterConfiguration("8.7.0"); -var cluster = new EphemeralCluster(config); + +var config = new EphemeralClusterConfiguration("8.7.0", XPack | Security | SSL); +using var cluster = new EphemeralCluster(config); + +var exitEvent = new ManualResetEvent(false); +Console.CancelKeyPress += (sender, eventArgs) => { + cluster.Dispose(); + eventArgs.Cancel = true; + exitEvent.Set(); +}; using var started = cluster.Start(); +exitEvent.WaitOne();