diff --git a/doc/configuration-puppetdb.md b/doc/configuration-puppetdb.md index 187efaa2..0a2f63ee 100644 --- a/doc/configuration-puppetdb.md +++ b/doc/configuration-puppetdb.md @@ -67,3 +67,11 @@ SSL support is enabled via any of the `--puppetdb-ssl-...` command line options - The CA certificate should be the public certificate of the CA that signed your PuppetDB server's certificate. This file can be found in `/etc/puppetlabs/puppetdb/ssl/ca.pem` on a PuppetDB server. Since this is a public certificate, it is safe (and recommended) to distribute this file to any clients that may connect to this PuppetDB instance. - The client keypair (key, certificate, and optionally password) should be generated individually for each client. You should NOT copy SSL keypairs from your PuppetDB server (or anywhere else) to your clients. If you are using `octocatalog-diff` on a system that is managed by Puppet, you may wish to use the same SSL credentials that the system uses to authenticate to Puppet. With recent versions of the Puppet agent, those certificates are found in `/etc/puppetlabs/puppet/ssl`. + +# Puppet Enterprise PuppetDB Package Inventory + +Puppet Enterprise customers have an optional package inventory feature which can be enabled. When this feature is enabled an inventory of all system packages +is performed and uploaded as a fact which is then processed and stored independently of the normal Facter data in PuppetDB. Most environments won't need +to replicate the package inventory facts for testing with Octocatalog-Diff but if you want the package inventory data (if present) to be included +in the facts retrieved from PuppetDB by Octocatalog-Diff you should specify the `--puppetdb-package-inventory` flag. When enabled, this flag will instruct +Octocatalog-Diff to retrieve any package data found for a node from PuppetDB and include it in the facts used during the Octocatalog-Diff compile. diff --git a/lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb b/lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb new file mode 100644 index 00000000..038728df --- /dev/null +++ b/lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# When pulling facts from PuppetDB in a Puppet Enterprise environment, also include +# the Puppet Enterprise Package Inventory data in the fact results, if available. +# Generally you should not need to specify this, but including the package inventory +# data will produce a more accurate set of input facts for environments using +# package inventory. +# @param parser [OptionParser object] The OptionParser argument +# @param options [Hash] Options hash being constructed; this is modified in this method. +OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_package_inventory) do + has_weight 150 + + def parse(parser, options) + parser.on('--[no-]puppetdb-package-inventory', 'Include Puppet Enterprise package inventory data, if found') do |x| + options[:puppetdb_package_inventory] = x + end + end +end diff --git a/lib/octocatalog-diff/facts/puppetdb.rb b/lib/octocatalog-diff/facts/puppetdb.rb index e75108b9..722ca98d 100644 --- a/lib/octocatalog-diff/facts/puppetdb.rb +++ b/lib/octocatalog-diff/facts/puppetdb.rb @@ -36,6 +36,7 @@ def self.fact_retriever(options = {}, node) exception_class = nil exception_message = nil obj_to_return = nil + packages = nil (retries + 1).times do begin result = puppetdb.get(uri) @@ -61,8 +62,48 @@ def self.fact_retriever(options = {}, node) exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})" end end - return obj_to_return unless obj_to_return.nil? - raise exception_class, exception_message + + raise exception_class, exception_message if obj_to_return.nil? + + return obj_to_return if puppetdb_api_version < 4 || (!options[:puppetdb_package_inventory]) + + (retries + 1).times do + begin + result = puppetdb.get("/pdb/query/v4/package-inventory/#{node}") + packages = {} + result.each do |pkg| + key = "#{pkg['package_name']}+#{pkg['provider']}" + # Need to handle the situation where a package has multiple versions installed. + # The _puppet_inventory_1 hash lists them separated by "; ". + if packages.key?(key) + packages[key]['version'] += "; #{pkg['version']}" + else + packages[key] = pkg + end + end + break + rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc + exception_class = OctocatalogDiff::Errors::FactSourceError + exception_message = "Package inventory retrieval failed (#{exc.class}) (#{exc.message})" + # This is not expected to occur, but we'll leave it just in case. A query to package-inventory + # for a non-existant node returns a 200 OK with an empty list of packages: + rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError + packages = {} + rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc + exception_class = OctocatalogDiff::Errors::FactRetrievalError + exception_message = "Package inventory retrieval failed for node #{node} from PuppetDB (#{exc.message})" + end + end + + raise exception_class, exception_message if packages.nil? + + unless packages.empty? + obj_to_return['values']['_puppet_inventory_1'] = { + 'packages' => packages.values.map { |pkg| [pkg['package_name'], pkg['version'], pkg['provider']] } + } + end + + obj_to_return end end end diff --git a/spec/octocatalog-diff/fixtures/facts/valid-packages.yaml b/spec/octocatalog-diff/fixtures/facts/valid-packages.yaml new file mode 100644 index 00000000..2fa9a0fa --- /dev/null +++ b/spec/octocatalog-diff/fixtures/facts/valid-packages.yaml @@ -0,0 +1,12 @@ +--- !ruby/object:Puppet::Node::Facts +name: rspec-node.xyz.github.net +values: + _timestamp: '2016-03-16 16:02:13 -0500' + apt_update_last_success: 1458162123 + architecture: amd64 + clientcert: rspec-node.github.net + datacenter: xyz + domain: xyz.github.net + fqdn: rspec-node.xyz.github.net + ipaddress: 10.20.30.40 + kernel: Linux diff --git a/spec/octocatalog-diff/fixtures/packages/valid-packages.json b/spec/octocatalog-diff/fixtures/packages/valid-packages.json new file mode 100644 index 00000000..f312befe --- /dev/null +++ b/spec/octocatalog-diff/fixtures/packages/valid-packages.json @@ -0,0 +1,14 @@ +[ + { + "certname": "valid-packages", + "package_name": "kernel", + "version": "3.2.1", + "provider": "yum" + }, + { + "certname": "valid-packages", + "package_name": "bash", + "version": "4.0.0", + "provider": "yum" + } +] diff --git a/spec/octocatalog-diff/mocks/puppetdb.rb b/spec/octocatalog-diff/mocks/puppetdb.rb index 4d37d7c7..23c002fc 100644 --- a/spec/octocatalog-diff/mocks/puppetdb.rb +++ b/spec/octocatalog-diff/mocks/puppetdb.rb @@ -24,6 +24,7 @@ def initialize(overrides = {}) def get(uri) return facts(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/nodes/([^/]+)/facts$} return catalog(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/catalogs/(.+)$} + return packages(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/package-inventory/(.+)$} raise ArgumentError, "PuppetDB URL not mocked: #{uri}" end @@ -63,6 +64,21 @@ def catalog(hostname) raise OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, '404 - Not Found' unless File.file?(fixture_file) JSON.parse(File.read(fixture_file)) end + + # Mock packages from PuppetDB + # @param hostname [String] Host name + # @return [String] JSON catalog + def packages(hostname) + fixture_file = OctocatalogDiff::Spec.fixture_path(File.join('packages', "#{hostname}.json")) + + # If packages are requested from PuppetDB for an invalid node name, it will return 200 OK + # with an empty list: + if File.file?(fixture_file) + JSON.parse(File.read(fixture_file)) + else + [] + end + end end end end diff --git a/spec/octocatalog-diff/tests/facts/puppetdb_spec.rb b/spec/octocatalog-diff/tests/facts/puppetdb_spec.rb index 9138a829..baf2ec53 100644 --- a/spec/octocatalog-diff/tests/facts/puppetdb_spec.rb +++ b/spec/octocatalog-diff/tests/facts/puppetdb_spec.rb @@ -11,7 +11,8 @@ allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| OctocatalogDiff::Mocks::PuppetDB.new } @opts = { puppetdb_url: 'https://mocked-puppetdb.somedomain.xyz:8081', - node: 'valid-facts' + node: 'valid-facts', + puppetdb_package_inventory: true } end @@ -22,6 +23,7 @@ expect(fact_obj).to be_a_kind_of(Hash) expect(fact_obj['name']).to eq(node) expect(fact_obj['values']['fqdn']).to eq('rspec-node.xyz.github.net') + expect(fact_obj['values']['_puppet_packages_1']).to be_nil end it 'should catch and handle error for non-existent host' do @@ -30,19 +32,43 @@ OctocatalogDiff::Facts::PuppetDB.fact_retriever(@opts, node) end.to raise_error(OctocatalogDiff::Errors::FactRetrievalError) end + + it 'should retrieve packages for the valid-packages node' do + node = 'valid-packages' + fact_obj = OctocatalogDiff::Facts::PuppetDB.fact_retriever(@opts, node) + expect(fact_obj).to be_a_kind_of(Hash) + expect(fact_obj['name']).to eq(node) + expect(fact_obj['values']['_puppet_inventory_1']).to be_a_kind_of(Hash) + expect(fact_obj['values']['_puppet_inventory_1']['packages']).to be_a_kind_of(Array) + expect(fact_obj['values']['_puppet_inventory_1']['packages']).to eq( + [ + ['kernel', '3.2.1', 'yum'], + ['bash', '4.0.0', 'yum'] + ] + ) + end end end context 'PuppetDB API compatibility layer' do before(:each) do clazz = double('OctocatalogDiff::PuppetDB') - allow(clazz).to receive(:get) { |args| [{ 'certname' => 'foo.bar.com', 'name' => 'uri', 'value' => args }] } + allow(clazz).to receive(:get) do |args| + if args =~ /package-inventory/ + packages + else + [{ 'certname' => 'foo.bar.com', 'name' => 'uri', 'value' => args }] + end + end allow(OctocatalogDiff::PuppetDB).to receive(:new).and_return(clazz) end + let(:packages) { [] } + it 'should use the correct URL for API v3' do opts = { puppetdb_api_version: 3, + puppetdb_package_inventory: true, puppetdb_url: 'https://foo.bar.baz:8081' } result = OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') @@ -52,6 +78,7 @@ it 'should use the correct URL for API v4' do opts = { puppetdb_api_version: 4, + puppetdb_package_inventory: true, puppetdb_url: 'https://foo.bar.baz:8081' } result = OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') @@ -60,6 +87,7 @@ it 'should default to API v4' do opts = { + puppetdb_package_inventory: true, puppetdb_url: 'https://foo.bar.baz:8081' } result = OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') @@ -69,42 +97,162 @@ it 'should fail if an unrecognized API version is provided' do opts = { puppetdb_api_version: 9000, + puppetdb_package_inventory: true, puppetdb_url: 'https://foo.bar.baz:8081' } expect { OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') }.to raise_error(KeyError) end + + context 'when packages returns data' do + let(:packages) do + [ + { + 'certname' => 'foo.bar.com', + 'package_name' => 'foo', + 'version' => '1.2.3', + 'provider' => 'yum' + }, + { + 'certname' => 'foo.bar.com', + 'package_name' => 'kernel', + 'version' => '3.2.1', + 'provider' => 'yum' + }, + { + 'certname' => 'foo.bar.com', + 'package_name' => 'kernel', + 'version' => '3.2.2', + 'provider' => 'yum' + } + ] + end + + it 'should return packages when puppetdb_package_inventory is enabled' do + opts = { + puppetdb_url: 'https://foo.bar.baz:8081', + puppetdb_package_inventory: true + } + result = OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') + expect(result).to eq( + 'name' => 'foo.bar.com', + 'values' => { + 'uri' => '/pdb/query/v4/nodes/foo.bar.com/facts', + '_puppet_inventory_1' => { + 'packages' => [ + ['foo', '1.2.3', 'yum'], + ['kernel', '3.2.1; 3.2.2', 'yum'] + ] + } + } + ) + end + + it 'should not return packages when puppetdb_package_inventory is false' do + opts = { + puppetdb_package_inventory: false, + puppetdb_url: 'https://foo.bar.baz:8081' + } + result = OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, 'foo.bar.com') + expect(result).to eq( + 'name' => 'foo.bar.com', + 'values' => { + 'uri' => '/pdb/query/v4/nodes/foo.bar.com/facts' + } + ) + end + end end context 'mocking methods for error testing' do describe '#fact_retriever' do - let(:opts) { { puppetdb_url: 'https://mocked-puppetdb.somedomain.xyz:8081', node: 'valid-facts' } } - let(:node) { 'valid-facts' } + context 'error during fact retrieval' do + let(:opts) { { puppetdb_url: 'https://mocked-puppetdb.somedomain.xyz:8081', node: 'valid-facts' } } + let(:node) { 'valid-facts' } - it 'should handle OctocatalogDiff::Errors::PuppetDBConnectionError' do - obj = double('OctocatalogDiff::PuppetDB') - allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBConnectionError, 'test message') - allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } - expect do - OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) - end.to raise_error(OctocatalogDiff::Errors::FactSourceError, /Fact retrieval failed \(.*ConnectionError\) \(test/) - end + it 'should handle OctocatalogDiff::Errors::PuppetDBConnectionError' do + obj = double('OctocatalogDiff::PuppetDB') + allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBConnectionError, 'test message') + allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } + expect do + OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) + end.to raise_error(OctocatalogDiff::Errors::FactSourceError, /Fact retrieval failed \(.*ConnectionError\) \(test/) + end - it 'should handle OctocatalogDiff::Errors::PuppetDBNodeNotFoundError' do - obj = double('OctocatalogDiff::PuppetDB') - allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, 'test message') - allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } - expect do - OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) - end.to raise_error(OctocatalogDiff::Errors::FactRetrievalError, /Node valid-facts not found in PuppetDB \(test/) + it 'should handle OctocatalogDiff::Errors::PuppetDBNodeNotFoundError' do + obj = double('OctocatalogDiff::PuppetDB') + allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, 'test message') + allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } + expect do + OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) + end.to raise_error(OctocatalogDiff::Errors::FactRetrievalError, /Node valid-facts not found in PuppetDB \(test/) + end + + it 'should handle OctocatalogDiff::Errors::PuppetDBGenericError' do + obj = double('OctocatalogDiff::PuppetDB') + allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBGenericError, 'test message') + allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } + expect do + OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) + end.to raise_error(OctocatalogDiff::Errors::FactRetrievalError, /Fact retrieval failed for node valid-facts/) + end end - it 'should handle OctocatalogDiff::Errors::PuppetDBGenericError' do - obj = double('OctocatalogDiff::PuppetDB') - allow(obj).to receive(:get).and_raise(OctocatalogDiff::Errors::PuppetDBGenericError, 'test message') - allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| obj } - expect do - OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) - end.to raise_error(OctocatalogDiff::Errors::FactRetrievalError, /Fact retrieval failed for node valid-facts/) + context 'error during package inventory retrieval' do + before(:each) do + allow(OctocatalogDiff::PuppetDB).to receive(:new) { |*_arg| puppetdb } + end + + let(:opts) do + { + puppetdb_url: 'https://mocked-puppetdb.somedomain.xyz:8081', + node: 'valid-packages', + puppetdb_package_inventory: true + } + end + let(:node) { 'valid-packages' } + let(:puppetdb) { OctocatalogDiff::Mocks::PuppetDB.new } + + it 'should handle OctocatalogDiff::Errors::PuppetDBConnectionError' do + allow(puppetdb).to receive(:get).and_wrap_original do |m, *args| + if args[0] =~ %r{/package-inventory/} + raise(OctocatalogDiff::Errors::PuppetDBConnectionError, 'test message') + else + m.call(*args) + end + end + + expect do + OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) + end.to raise_error( + OctocatalogDiff::Errors::FactSourceError, /Package inventory retrieval failed \(.*ConnectionError\) \(test/ + ) + end + + it 'should handle OctocatalogDiff::Errors::PuppetDBNodeNotFoundError' do + allow(puppetdb).to receive(:get).and_wrap_original do |m, *args| + if args[0] =~ %r{/package-inventory/} + raise(OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, 'test message') + else + m.call(*args) + end + end + expect(OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node)).to be_a_kind_of(Hash) + end + + it 'should handle OctocatalogDiff::Errors::PuppetDBGenericError' do + allow(puppetdb).to receive(:get).and_wrap_original do |m, *args| + if args[0] =~ %r{/package-inventory/} + raise(OctocatalogDiff::Errors::PuppetDBGenericError, 'test message') + else + m.call(*args) + end + end + expect do + OctocatalogDiff::Facts::PuppetDB.fact_retriever(opts, node) + end.to raise_error( + OctocatalogDiff::Errors::FactRetrievalError, /Package inventory retrieval failed for node valid-packages/ + ) + end end end end