From 56f052d0e096d7c964aa161481b9f9aec962d8f2 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:38:51 -0800 Subject: [PATCH 01/15] Account for stopping states when determining existing vas vms may be in the stopping or destroyed state should not be treated as present. --- lib/puppet/provider/cloudstack_instance/default.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/puppet/provider/cloudstack_instance/default.rb b/lib/puppet/provider/cloudstack_instance/default.rb index 5ba17b4..e66a20c 100644 --- a/lib/puppet/provider/cloudstack_instance/default.rb +++ b/lib/puppet/provider/cloudstack_instance/default.rb @@ -9,7 +9,10 @@ def self.instances # I may need to fail if the server does not have a name? connection.servers.collect do |server| - if server.state != 'Destroyed' + #require 'ruby-debug';debugger + if (server.state != 'Destroyed') and (server.state != 'Stopping') + Puppet.debug("Found #{server.display_name} in state: #{server.state}") + Puppet.debug("Found #{server.display_name} in state: #{server.state}") if server.nics.size > 1 raise(Puppet::Error, "Does not support dual nics (it is just a prototype") end From e340bc42772dc7a704fd9f525cf80116a79293a7 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:40:06 -0800 Subject: [PATCH 02/15] update instance property hash so that subsequent objects can inspect it. --- lib/puppet/provider/cloudstack_instance/default.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/puppet/provider/cloudstack_instance/default.rb b/lib/puppet/provider/cloudstack_instance/default.rb index e66a20c..9621f51 100644 --- a/lib/puppet/provider/cloudstack_instance/default.rb +++ b/lib/puppet/provider/cloudstack_instance/default.rb @@ -61,11 +61,14 @@ def create :group => resource[:group] #:keypair => resource[:keypair] ) + @property_hash[:ensure] = :present + @property_hash[:internal_ipaddress] = response.nics.first['ipaddress'] end def destroy # TODO need to block until delete is completed - connection.servers.destroy(@property_hash[:id]) + connection.servers.destroy(@property_hash[:id]) + @property_hash[:ensure] = :absent end def internal_ipaddress From 2323034468ae935063b06cb1c2d2f2104bfb4564 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:40:32 -0800 Subject: [PATCH 03/15] Add key pair as instance property --- lib/puppet/provider/cloudstack_instance/default.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/puppet/provider/cloudstack_instance/default.rb b/lib/puppet/provider/cloudstack_instance/default.rb index 9621f51..f60750f 100644 --- a/lib/puppet/provider/cloudstack_instance/default.rb +++ b/lib/puppet/provider/cloudstack_instance/default.rb @@ -29,6 +29,8 @@ def self.instances :host => server.host_name, :state => server.state.downcase, :group => server.group, + # for some reason the keypair does not seem to be returned from listvminstnaces + :keypair => server.key_pair, #:keypair => server.keypair, :ensure => :present # I may want to print network information here @@ -50,14 +52,16 @@ def create :image_id => #{image_id}, :flavor_id => #{flavor_id}, :zone_id => #{zone_id}, - :network_ids => #{network_id} + :network_ids => #{network_id}, + :keypair => #{resource[:keypair]}, ") - connection.servers.bootstrap( + response = connection.servers.bootstrap( :display_name => resource[:name], :image_id => image_id, :flavor_id => flavor_id, :zone_id => zone_id, :network_ids => network_id, + :key_pair => resource[:keypair], :group => resource[:group] #:keypair => resource[:keypair] ) From 6efdb406c18ba115716adb9d07d0f2ebb6398de3 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:40:47 -0800 Subject: [PATCH 04/15] add uesrdata as instance parameter --- lib/puppet/type/cloudstack_instance.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/puppet/type/cloudstack_instance.rb b/lib/puppet/type/cloudstack_instance.rb index 04cd1d3..cd93e4e 100644 --- a/lib/puppet/type/cloudstack_instance.rb +++ b/lib/puppet/type/cloudstack_instance.rb @@ -78,6 +78,10 @@ def insync?(is) end end + newparam(:userdata) do + desc 'used to set userdata when an instance is created' + end + # this causes querying the resources to fail ;( #validate do From 18346ca52cf1605ac46829d5c45426ddf8eda280 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:41:15 -0800 Subject: [PATCH 05/15] chmod of cached private keys to 600 so that ssh will use them by default --- lib/puppet/provider/cloudstack_keypair/default.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/puppet/provider/cloudstack_keypair/default.rb b/lib/puppet/provider/cloudstack_keypair/default.rb index 950c1dc..f4c994a 100644 --- a/lib/puppet/provider/cloudstack_keypair/default.rb +++ b/lib/puppet/provider/cloudstack_keypair/default.rb @@ -41,6 +41,7 @@ def create end Puppet.info("Writing your private key to #{key_file}") File.new(key_file, 'w').write(response['privatekey']) + File.chmod(0600, key_file) end @property_hash[:fingerprint] = response['fingerprint'] @property_hash[:privatekey] = response['privatekey'] From 53f8c2ca0bf2064a9acd4f993f69d93ac9672d1f Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:41:54 -0800 Subject: [PATCH 06/15] do not fail if there are not network just adds some extra checking to prevent nil exceptions when there are no networks defined. --- lib/puppet/provider/cloudstack_network/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/puppet/provider/cloudstack_network/default.rb b/lib/puppet/provider/cloudstack_network/default.rb index e8c8fb5..663b2e4 100644 --- a/lib/puppet/provider/cloudstack_network/default.rb +++ b/lib/puppet/provider/cloudstack_network/default.rb @@ -9,7 +9,7 @@ mk_resource_methods def self.instances - connection.list_networks['listnetworksresponse']['network'].collect do |net| + (connection.list_networks['listnetworksresponse']['network'] || []).collect do |net| new( :name => net['name'], :id => net['id'], From d0078ec8af3c270f7996cdafe0b4c21a56e81d46 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:42:34 -0800 Subject: [PATCH 07/15] do not fail if there are no key pairs Adds extra error checking to prevent failures when there are not key pairs defined. --- lib/puppet/provider/cloudstack_keypair/default.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/puppet/provider/cloudstack_keypair/default.rb b/lib/puppet/provider/cloudstack_keypair/default.rb index f4c994a..0f04564 100644 --- a/lib/puppet/provider/cloudstack_keypair/default.rb +++ b/lib/puppet/provider/cloudstack_keypair/default.rb @@ -8,8 +8,7 @@ def self.instances # I may need to fail if the server does not have a name? - connection.list_ssh_key_pairs['listsshkeypairsresponse']['sshkeypair'].collect do |keypair| - #require 'ruby-debug';debugger + (connection.list_ssh_key_pairs['listsshkeypairsresponse']['sshkeypair'] || []).collect do |keypair| new( :name => keypair['name'], :fingerprint => keypair['fingerprint'], From da187ba69d53e57267018dbfccf9dd251cd1a600 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:43:44 -0800 Subject: [PATCH 08/15] do not set property_hash to resource in flush b/c some of the properties being set are not set in the resource, but parsed from the creation response. --- lib/puppet/provider/cloudstack_keypair/default.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/puppet/provider/cloudstack_keypair/default.rb b/lib/puppet/provider/cloudstack_keypair/default.rb index 0f04564..8642763 100644 --- a/lib/puppet/provider/cloudstack_keypair/default.rb +++ b/lib/puppet/provider/cloudstack_keypair/default.rb @@ -67,8 +67,8 @@ def key_file_path(fingerprint_arg=fingerprint) File.join(Puppet[:confdir], 'cloudstack', 'keypair', fingerprint_arg, 'id_rsa') end - def flush - @property_hash = resource.to_hash - end +# def flush +# @property_hash = resource.to_hash +# end -end \ No newline at end of file +end From b60506504af2516071c56c9779909a11de35f889 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:44:21 -0800 Subject: [PATCH 09/15] add error checking when key file is searched for. --- lib/puppet/provider/cloudstack_keypair/default.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/puppet/provider/cloudstack_keypair/default.rb b/lib/puppet/provider/cloudstack_keypair/default.rb index 8642763..6a755ef 100644 --- a/lib/puppet/provider/cloudstack_keypair/default.rb +++ b/lib/puppet/provider/cloudstack_keypair/default.rb @@ -53,7 +53,13 @@ def destroy def privatekey if fingerprint - File.read(key_file_path(fingerprint)) + key_file = key_file_path(fingerprint) + if File.exists?(key_file) + File.read(key_file) + else + Puppet.notice("Could not find locally cached private key for #{resource[:name]}") + nil + end else fail('Expected fingerprint to be set') end From e37b74a3ea49b61b082411bea1b253e8e4c1b974 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:45:35 -0800 Subject: [PATCH 10/15] Expand puppet_node resource This resource supports installing puppet enterprise, and classifying agents. --- lib/puppet/provider/puppet_node/ssh.rb | 103 +++++++++++++- lib/puppet/type/puppet_node.rb | 30 +++- templates/agent.erb | 11 ++ templates/apply.erb | 13 ++ templates/check_class_facts.erb | 12 ++ templates/check_puppet_version.erb | 8 ++ templates/fragments/install_pe.erb | 52 +++++++ templates/fragments/install_poss.erb | 88 ++++++++++++ templates/fragments/options.erb | 59 ++++++++ templates/fragments/set_http_get.erb | 10 ++ templates/master.erb | 12 ++ templates/pe_agent.erb | 11 ++ templates/pe_master.erb | 9 ++ templates/puppet-community.sh | 188 +++++++++++++++++++++++++ 14 files changed, 597 insertions(+), 9 deletions(-) create mode 100644 templates/agent.erb create mode 100644 templates/apply.erb create mode 100644 templates/check_class_facts.erb create mode 100644 templates/check_puppet_version.erb create mode 100644 templates/fragments/install_pe.erb create mode 100644 templates/fragments/install_poss.erb create mode 100644 templates/fragments/options.erb create mode 100644 templates/fragments/set_http_get.erb create mode 100644 templates/master.erb create mode 100644 templates/pe_agent.erb create mode 100644 templates/pe_master.erb create mode 100644 templates/puppet-community.sh diff --git a/lib/puppet/provider/puppet_node/ssh.rb b/lib/puppet/provider/puppet_node/ssh.rb index 0ba5bde..8877204 100644 --- a/lib/puppet/provider/puppet_node/ssh.rb +++ b/lib/puppet/provider/puppet_node/ssh.rb @@ -10,16 +10,77 @@ # end def role - # make an ssh call? decide if the role is already installed? - false + # ssh into the instance to see if puppet is already installed + # TODO eventually, this should also check the version + begin + results = run_remote_puppet_script('check_puppet_version') + rescue Puppet::Error => e + return nil + end + version = results['stdout'].split("\n").last.strip + Puppet.info("Puppet version #{version} is currently installed") + version end def role=(role) - machinehostname(resource[:machine]) - getprivatekey(resource[:keypair]) - if role == :master + if role == :pe_master + script_name = 'pe_master' + elsif role == :pe_agent + script_name = 'pe_agent' else + fail("unsupported role #{role.to_s}") end + run_remote_puppet_script(script_name) + end + + def run_remote_puppet_script(script_name) + + hostname = machinehostname(resource[:machine]) + keyfile = getprivatekey(resource[:keypair]) + + if script_name == 'pe_master' + answers = 'https://raw.github.com/gist/4229037/d03453ae789797909745b63d44611adc5090c050/gistfile1.txt' + elsif script_name == 'pe_agent' + answers = 'https://raw.github.com/gist/3299812/9dea06dba93d218c61e5fa9d9e928a265c137239/answers' + else + Puppet.debug("Script name: #{script_name} does not need an answers file") + end + + Puppet.info("Running script #{script_name} via ssh to '#{hostname}' with key: '#{keyfile}'") + + write_name = "#{hostname}-#{Time.now.to_i}" + unless File.exists?(script_write_dir) + Puppet.info('Creating script dir') + Dir.mkdir(script_write_dir) + end + + compile_options = { + 'installer_payload' => resource[:installer_payload], + 'answers_payload' => answers, + 'certname' => machinehostname(resource[:machine]), + 'puppetmaster' => machinehostname(resource[:puppetmaster]), + 'facts' => 'classes' => resource['classes'].to_pson + } + + Puppet.debug("Compiling script #{script_name} with options:\n#{compile_options.inspect}") + + File.open(File.join(script_write_dir, "#{write_name}.erb"), 'w') do |fh| + fh.write( + compile_erb(File.join(script_dir, "#{script_name}.erb"), compile_options) + ) + end + + require 'puppet/cloudpack' + + Puppet.info("running: ssh root@#{hostname} -i '#{keyfile}'") + Puppet::CloudPack.install(hostname, + { + :install_script => write_name, + :keyfile => keyfile, + :login => 'root' + + } + ) end def modules @@ -27,6 +88,14 @@ def modules end def getprivatekey(key) + fail('Must specify a private key') unless key + key = key.to_s + if key =~ /(Cloudstack_keypair\[(\S+)?\])/ + return model.catalog.resource($1).provider.key_file_path + else + puts 'It is a path' + return key + end key end @@ -41,10 +110,14 @@ def machinehostname(id) raise(Puppet::Error, "Do not support things other than internal_ip at the moment") else # assume we want the internal ip address - res.provider.internal_ipaddress + if res.provider.exists? + res.provider.internal_ipaddress + else + fail("Referenced cloudstack instance #{$1} does not exist") + end end else - raise(Puppet::Error, "Puppet resource #{$1} was set at the machine, but not found in the catalog, try hostname instead") + fail("Puppet resource #{$1} was set at the machine, but not found in the catalog, try hostname instead") end else # otherwise assume id is the hostname of ip address @@ -52,4 +125,20 @@ def machinehostname(id) end end + def compile_erb(filename, options) + ERB.new(File.read(filename)).result(binding) + end + + def script_write_dir + File.join(Puppet[:confdir], 'scripts') + end + + def script_dir + File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', '..', 'templates')) + end + + def find_template(name) + File.expand_path(File.join(script_dir, "#{name}.erb")) + end + end diff --git a/lib/puppet/type/puppet_node.rb b/lib/puppet/type/puppet_node.rb index 9baa383..995906f 100644 --- a/lib/puppet/type/puppet_node.rb +++ b/lib/puppet/type/puppet_node.rb @@ -24,13 +24,39 @@ newproperty(:role) do desc 'role of this puppet instance' - newvalues(:agent, :master) + newvalues(:agent, :master, :pe_agent, :pe_master, :apply) + def insync?(is) + return is =~ /^\d+\.\d+\.\d+/ + end + end + + newparam(:installer_payload) do + # right now this is hardcoded to rhel 5 pe installation payload + defaultto 'https://s3.amazonaws.com/pe-builds/released/2.6.1/puppet-enterprise-2.6.1-el-5-x86_64.tar.gz' + end + + newparam(:answers_payload) do + defaultto 'https://raw.github.com/gist/4229037/d03453ae789797909745b63d44611adc5090c050/gistfile1.txt' + end + + newparam(:puppetmaster) do + desc 'hostname of puppetmaster to connect to. Used by pe_agent and agent roles.' + end + + newparam(:facts) do + desc 'hash of facts that can be used to set instance role' + defaultto {} end autorequire(:cloudstack_instance) do + instances = [] if self[:machine].to_s =~ /Cloudstack_instance\[(\S+?)\](\[\S+?\])?/ - [$1] + instances.push($1) + end + if self[:puppetmaster].to_s =~ /Cloudstack_instance\[(\S+?)\](\[\S+?\])?/ + instances.push($1) end + instances end end diff --git a/templates/agent.erb b/templates/agent.erb new file mode 100644 index 0000000..792e2c1 --- /dev/null +++ b/templates/agent.erb @@ -0,0 +1,11 @@ +#!/bin/bash +# +# script that performs classification using puppet agent +# + +set -u +set -e + +<%= ERB.new(File.read(find_template('fragments/options')), nil, "<>", '_options').result(binding) %> + +puppet agent --pluginsync --test --certname=<%= options['certname'] %> --server=<%= options['puppetmaster'] %> | tee /tmp/puppet_output.log diff --git a/templates/apply.erb b/templates/apply.erb new file mode 100644 index 0000000..484d358 --- /dev/null +++ b/templates/apply.erb @@ -0,0 +1,13 @@ +#!/bin/bash +# +# script that performs classification using puppet apply +# + +set -u +set -e + +<% if options['install_puppet'] %> +<%= ERB.new(File.read(find_template('fragments/install_poss')), nil, "<>", '_install_poss').result(binding) %> +<% end %> + +<%= ERB.new(File.read(find_template('fragments/install_poss')), nil, "<>", '_install_poss').result(binding) %> diff --git a/templates/check_class_facts.erb b/templates/check_class_facts.erb new file mode 100644 index 0000000..6318a0f --- /dev/null +++ b/templates/check_class_facts.erb @@ -0,0 +1,12 @@ +#!/bin/sh +set -x +set -e +set -u + +echo 'Checking if pe is installed' + +if [ -f /etc/facts.d/classes]; then + cat /etc/facts.d/classes.txt +else + exit 1 +fi diff --git a/templates/check_puppet_version.erb b/templates/check_puppet_version.erb new file mode 100644 index 0000000..bc65e50 --- /dev/null +++ b/templates/check_puppet_version.erb @@ -0,0 +1,8 @@ +#!/bin/sh +set -x +set -e +set -u + +echo 'Checking if pe is installed' + +puppet --version diff --git a/templates/fragments/install_pe.erb b/templates/fragments/install_pe.erb new file mode 100644 index 0000000..b3bfe20 --- /dev/null +++ b/templates/fragments/install_pe.erb @@ -0,0 +1,52 @@ +<%= ERB.new(File.read(find_template('fragments/set_http_get')), nil, "<>", '_ipe_set_http_get').result(binding) %> + +PATH="/opt/puppet/bin:$PATH" +export PATH + +install_dir="puppet-enterprise" +if [ ! -e "${install_dir}" ]; then + mkdir "${install_dir}" +fi + +echo "Downloading from: <%= options['installer_payload'] %>" + +# We assume the payload is a tar.gz file because the option handler +# enforces this. +# To save disk space (I'm concerned about /tmp filling) decompress on the fly. +echo "Uncompressing the payload ..." +$http_get '<%= options['installer_payload'] %>' | \ + gunzip -c | \ + { cd "${install_dir}" && tar -xvf -; } && \ + echo "Uncompressing the payload ... Done." + +echo 'Downloading answers file' +$http_get '<%= options['answers_payload'] %>' > puppet.answers.orig + +# Give me everything _except_ any agent certname specified in the answers +# file, and the agent's server setting. Useless use of cat is for clarity +# Note that this particular ERB syntax requires that the trim_mode be set +# to <> when ERB.new is called (or some other alternative which will +# suppress newlines) or else this shell code will not work. +cat puppet.answers.orig | \ + grep -v '^q_puppetmaster_certname' | \ +<% if options['puppetmaster'] %> + grep -v '^q_puppetagent_server' | \ +<% end %> + cat > puppet.answers + +# Append the user specified option from the command line arguments. +echo 'q_puppetmaster_certname=<%= options['certname'] %>' >> puppet.answers +echo 'q_puppetagent_certname=<%= options['certname'] %>' >> puppet.answers +<% if options['puppetmaster'] %> +echo 'q_puppetagent_server=<%= options['puppetmaster'] %>' >> puppet.answers +<% end %> + +# Install Puppet Enterprise +"${install_dir}"/*puppet*/puppet-enterprise-installer -a puppet.answers 2>&1 | tee install.log + +echo '*' > /etc/puppetlabs/puppet/autosign.conf + +# if redhat turn off the firewall +if [ -f /etc/redhat-release ]; then + /etc/init.d/iptables stop +fi diff --git a/templates/fragments/install_poss.erb b/templates/fragments/install_poss.erb new file mode 100644 index 0000000..67cfd24 --- /dev/null +++ b/templates/fragments/install_poss.erb @@ -0,0 +1,88 @@ +fedora_repo() +{ + cat >/etc/yum.repos.d/puppet.repo <<-EOFYUMREPO + [puppetlabs] + name = Puppetlabs + baseurl = http://yum.puppetlabs.com/fedora/f$releasever/products/$basearch/ + gpgcheck = 1 + enabled = 1 + gpgkey = http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs + EOFYUMREPO +} + +el_repo() +{ + cat >/etc/yum.repos.d/puppet.repo <<-EOFYUMREPO + [puppetlabs] + name = Puppetlabs + baseurl = http://yum.puppetlabs.com/el/$releasever/products/$basearch/ + gpgcheck = 1 + enabled = 1 + gpgkey = http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs + EOFYUMREPO +} + +rpm_install() +{ + # Setup the yum Puppet repository + rpm -q fedora-release && fedora_repo || el_repo + + # Install git + yum install -y git + + # Install Puppet from yum.puppetlabs.com + yum install -y puppet +} + +apt_install() +{ + # Download and install the puppetlabs apt public + apt-key adv --recv-key --keyserver pool.sks-keyservers.net 4BD6EC30 + + # We need to grab the distro and release in order to populate + # the apt repo details. We are assuming that the lsb_release command + # will be available as even puppet evens has it (lsb_base) package as + # dependancy. + + # Since puppet requires lsb-release I believe this is ok to use for + # the purpose of distro and release discovery. + apt-get update + apt-get -y install git-core + apt-get -y install lsb-release + distro=$(lsb_release -i | cut -f 2 | tr "[:upper:]" "[:lower:]") + release=$(lsb_release -c | cut -f 2) + + # Setup the apt Puppet repository + cat > /etc/apt/sources.list.d/puppetlabs.list <<-EOFAPTREPO + deb http://apt.puppetlabs.com/${distro}/ ${release} main + EOFAPTREPO + + # Install Puppet from Debian repositories + apt-get update + apt-get -y install puppet + apt-get install -y git-core + apt-get install -y rubygems +} + +install_puppet() +{ + case "${osfamily}" in + "redhat") + rpm_install ;; + "debian") + apt_install ;; + esac +} + +# install puppet +if [ -f /etc/redhat-release ]; then + export osfamily='redhat' +elif [ -f /etc/debian_version ]; then + export osfamily='debian' +else + echo "This OS is not supported by Puppet Cloud Provisioner" + exit 1 +fi + +install_puppet +echo "Puppet installation finished!" diff --git a/templates/fragments/options.erb b/templates/fragments/options.erb new file mode 100644 index 0000000..04c84c7 --- /dev/null +++ b/templates/fragments/options.erb @@ -0,0 +1,59 @@ +<%= ERB.new(File.read(find_template('fragments/set_http_get')), nil, "<>", '_o_set_http_get').result(binding) %> + +# install git and rubygems +apt-get update || true +apt-get install -y git-core || yum install -y git +apt-get install -y rubygems || yum install -y rubygems + +# clone git repos +<% (options['git_repos'] || {}).each do |source, target| %> +mkdir -p <%= target %> || true +git clone <%= source %> <%= target %> +<% end %> + +# switch into specified branches +<% (options['git_checkout'] || {}).each do |dir, branch| %> +pushd <%= dir %> +git checkout <%= branch %> +popd +<% end %> + +# cp files from modules to system locations +<% (options['cp_files'] || {}).each do |source, target| %> +cp <%= source %> <%= target %> +<% end %> + +# if a manifest was specified apply it +<% if options['manifest'] %> +mkdir -p /var/lib/puppet_client +# <%= options.inspect %> +puppet apply --vardir /var/lib/puppet_client --trace --verbose <%= options['manifest'] %> --certname=<%= options['certname'] %> | tee /tmp/puppet_output.log +return=$? +<% end %> + +#This fuction is not called if no custom facts are given +# +#The Puppet Labs' stdlib module is required for this to work +#It can be installed using: +# 'puppet-module install puppetlabs/stdlib' +mkdir -p /etc/facter/facts.d +<% if options[:facts] %> +echo "Installing custom facts" +<% options[:facts].each do |fact,value| %> +echo <%= fact %>=<%= value %> > /etc/facter/facts.d/<%= fact %>.txt +<% end %> +<% end %> + +# upload and run postinstall scripts +<% [options['postinstall']].flatten.compact.each do |script| %> +POSTINSTALL=/tmp/postinstall +<% if script.match(%r{[a-zA-Z0-9]://}) %> +$http_get '<%= script %>' > $POSTINSTALL +<% else %> +cat > "$POSTINSTALL" <<-EOF_STACKBUILDER_POSTINSTALL +<%= heredoc_safe(File.read(script)) %> +EOF_STACKBUILDER_POSTINSTALL +<% end %> +chmod a+x "$POSTINSTALL" +"$POSTINSTALL" +<% end %> diff --git a/templates/fragments/set_http_get.erb b/templates/fragments/set_http_get.erb new file mode 100644 index 0000000..b09bcc4 --- /dev/null +++ b/templates/fragments/set_http_get.erb @@ -0,0 +1,10 @@ +# Figure out if we have wget or curl +if which wget >/dev/null; then + http_get="wget --no-check-certificate --output-document=- --quiet" +elif which curl >/dev/null; then + http_get="curl -o -" +else + echo "Error: could not find wget or curl on the remote system." + echo "To install Puppet Enterprise using HTTP, wget or curl are required" + exit 1 +fi diff --git a/templates/master.erb b/templates/master.erb new file mode 100644 index 0000000..64fb8a0 --- /dev/null +++ b/templates/master.erb @@ -0,0 +1,12 @@ +#!/bin/bash +# +# script that builds a puppet master using puppet apply +# + +set -u +set -e + +mkdir -p /etc/puppet/modules +pushd /etc/puppet/modules + +<%= ERB.new(File.read(find_template('fragments/options')), nil, "<>", '_options').result(binding) %> diff --git a/templates/pe_agent.erb b/templates/pe_agent.erb new file mode 100644 index 0000000..361ad80 --- /dev/null +++ b/templates/pe_agent.erb @@ -0,0 +1,11 @@ +#! /bin/bash +set -x +set -e +set -u + +echo "Installation script has started..." + +<%= ERB.new(File.read(find_template('fragments/install_pe')), nil, "<>", '_install_pe').result(binding) %> +<%= ERB.new(File.read(find_template('fragments/options')), nil, "<>", '_options').result(binding) %> + +puppet agent --pluginsync --test --certname=<%= options['certname'] %> --server=<%= options['puppetmaster'] %> | tee /tmp/puppet_output.log diff --git a/templates/pe_master.erb b/templates/pe_master.erb new file mode 100644 index 0000000..a651339 --- /dev/null +++ b/templates/pe_master.erb @@ -0,0 +1,9 @@ +#!/bin/sh +set -x +set -e +set -u + +echo "Installation script has started..." + +<%= ERB.new(File.read(find_template('fragments/install_pe')), nil, "<>", '_install_pe').result(binding) %> +<%= ERB.new(File.read(find_template('fragments/options')), nil, "<>", '_options').result(binding) %> diff --git a/templates/puppet-community.sh b/templates/puppet-community.sh new file mode 100644 index 0000000..08e8de3 --- /dev/null +++ b/templates/puppet-community.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# this script can be used in combination +# with the gce types to install puppet and +# classify the provisioned instances + +set -u +set -e + +RESULTS_FILE='/tmp/puppet_bootstrap_output' + +function check_exit_status() { + if [ ! -f $RESULTS_FILE ]; then + echo '1' > $RESULTS_FILE + fi +} + +trap check_exit_status INT TERM EXIT + +function fedora_repo() { + cat >/etc/yum.repos.d/puppet.repo <<'EOFYUMREPO' +[puppetlabs] +name = Puppetlabs +baseurl = http://yum.puppetlabs.com/fedora/f$releasever/products/$basearch/ +gpgcheck = 1 +enabled = 1 +gpgkey = http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs +EOFYUMREPO +} + +function el_repo() { + cat >/etc/yum.repos.d/puppet.repo <<'EOFYUMREPO' +[puppetlabs] +name = Puppetlabs +baseurl = http://yum.puppetlabs.com/el/$releasever/products/$basearch/ +gpgcheck = 1 +enabled = 1 +gpgkey = http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs +[puppetlabs-deps] +name = Puppetlabs Dependencies +baseurl = http://yum.puppetlabs.com/el/$releasever/dependencies/$basearch/ +gpgcheck = 1 +enabled = 1 +gpgkey = http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs +EOFYUMREPO +} + +function rpm_install() { + # Setup the yum Puppet repository + rpm -q fedora-release && fedora_repo || el_repo + + # Install Puppet from yum.puppetlabs.com + yum install -y puppet git +} + +function apt_install() { + # Download and install the puppetlabs apt public + apt-key adv --recv-key --keyserver pool.sks-keyservers.net 4BD6EC30 + + # We need to grab the distro and release in order to populate + # the apt repo details. We are assuming that the lsb_release command + # will be available as even puppet evens has it (lsb_base) package as + # dependancy. + + # Since puppet requires lsb-release I believe this is ok to use for + # the purpose of distro and release discovery. + apt-get update + apt-get -y install lsb-release + distro=$(lsb_release -i | cut -f 2 | tr "[:upper:]" "[:lower:]") + release=$(lsb_release -c | cut -f 2) + + # Setup the apt Puppet repository + cat > /etc/apt/sources.list.d/puppetlabs.list </etc/puppet/puppet.conf < /etc/default/puppet < /etc/puppet/manifests/empty.pp + echo '#!/bin/bash + cat /etc/puppet/nodes/$1.yaml' > /etc/puppet/nodes/enc.sh + chmod a+x /etc/puppet/nodes/enc.sh + echo "$1" > /etc/puppet/nodes/"$2".yaml + # yaml terminus does not merge facts, so it failed with puppet + # apply + puppet apply --trace --debug --node_terminus=exec --external_nodes=/etc/puppet/nodes/enc.sh /etc/puppet/manifests/empty.pp + fi +} + + +function provision_puppet() { + if [ -f /etc/redhat-release ]; then + export breed='redhat' + elif [ -f /etc/debian_version ]; then + export breed='debian' + else + echo "This OS is not supported by Puppet Cloud Provisioner" + exit 1 + fi + + PUPPET_CLASSES=$(curl http://metadata.google.internal/0.1/meta-data/attributes/puppet_classes) + # BEGIN HACK + # + # This is a pretty awful hack, but I did not really understand a better way to do it. + # The problem is that applications may need to specify facts or other system specific information + # as a part of the classifaction process. I this case, I need to be able to figure out my own internal + # and external ip addresses. + # I am going to just pass in these specific things as variables in the puppetcode and parse them out here. + # Eventually, I may want to do some kind of a fact lookup + GCE_EXTERNAL_IP=$(curl http://metadata.google.internal/0.1/meta-data/network | tr ":" "\n" | grep -A 1 externalIp | tail -1 | cut -f 2 -d '"') + GCE_INTERNAL_IP=$(curl http://metadata.google.internal/0.1/meta-data/network | tr ":" "\n" | grep -A 1 ip | tail -1 | cut -f 2 -d '"') + PUPPET_CLASSES=$(echo "$PUPPET_CLASSES" | sed -e "s/\$gce_external_ip/$GCE_EXTERNAL_IP/" -e "s/\$gce_internal_ip/$GCE_INTERNAL_IP/") + # END HACK + PUPPET_MODULES=$(curl http://metadata.google.internal/0.1/meta-data/attributes/puppet_modules) + PUPPET_REPOS=$(curl http://metadata.google.internal/0.1/meta-data/attributes/puppet_repos) + PUPPET_HOSTNAME=$(curl http://metadata.google.internal/0.1/meta-data/hostname) + + install_puppet + configure_puppet "$PUPPET_HOSTNAME" + download_modules "$PUPPET_MODULES" + clone_modules "$PUPPET_REPOS" + run_puppet_apply "$PUPPET_CLASSES" "$PUPPET_HOSTNAME" + echo $? > $RESULTS_FILE + echo "Puppet installation finished!" + exit 0 +} + +provision_puppet From 1430bd3fb3313ce803c83826a091be40e09b8437 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Thu, 13 Dec 2012 14:45:55 -0800 Subject: [PATCH 11/15] update example manifest to demonstrate creation of supported objects. --- tests/example.pp | 105 +++++++++++++++++++++++------------------------ 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/tests/example.pp b/tests/example.pp index 189c8f2..c93e9b9 100644 --- a/tests/example.pp +++ b/tests/example.pp @@ -1,68 +1,65 @@ -# -# transport object, manages connections to cloudstack API server -# -#cloudstack_transport { 'transport1': } - -# this requires advanced zones -#cloudstack_firewall_rule { -# -#} - -#cloud_portforward_rule { -# -#} - -# this does not - -# I need a basic zone to deploy this into - -#cloudstack_security_group { 'foo': -# ensure => present, -#} - -#volume { -# -#} +if (($::ensure != 'present') and ($::ensure != 'absent')) { + fail('invalid ensure value') +} resources { 'cloudstack_instance': purge => true } -# ssh_transport { -# -# } - -cloudstack_instance { 'foo1': - ensure => present, - flavor => 'Small Instance', - zone => 'FMT-ACS-001', - image => 'CentOS 5.6(64-bit) no GUI (XenServer)', +Cloudstack_instance { + image => 'CentOS 5.6 key + pw', + flavor => 'Large Local', + zone => 'ACS-FMT-1', + #zone => 'FMT-ACS-001', network => 'puppetlabs-network', - # domain - # account - # hostname + keypair => 'dans_keypair4', + require => Cloudstack_keypair['dans_keypair4'], } -cloudstack_instance { 'foo2': - ensure => present, - flavor => 'Small Instance', - zone => 'FMT-ACS-001', - image => 'CentOS 5.6(64-bit) no GUI (XenServer)', - network => 'puppetlabs-network', - group => 'foo2', +cloudstack_keypair { 'dans_keypair4': + ensure => $::ensure, + cache_key => true, } +cloudstack_instance { 'foo3': + ensure => $::ensure, + group => 'role=puppetmaster', } +cloudstack_instance { 'foo4': + ensure => $::ensure, + group => 'role=db', +} -#cloudstack_ssh { -# classes => {} -# modules => [] -# ssh_transport => -# machine => Cloudstack_instnace['foo1'], -# type => 'agent' -#} +cloudstack_instance { 'foo5': + ensure => $::ensure, + userdata => 'foo', +} -# -# should I create a domain? -# +if $::ensure != 'absent' { + puppet_node { 'puppet_master': + role => 'pe_master', + machine => "Cloudstack_instance[foo3]", + keypair => Cloudstack_keypair['dans_keypair4'], + require => Cloudstack_keypair['dans_keypair4'], + #machine => Cloudstack_instance['foo1'], + } + + puppet_node { 'puppet_agent1': + role => 'pe_agent', + machine => "Cloudstack_instance[foo4]", + keypair => Cloudstack_keypair['dans_keypair4'], + require => [Cloudstack_keypair['dans_keypair4'], Puppet_node['puppet_master']], + puppetmaster => "Cloudstack_instance[foo3]", + classes => {'apache' => {}} + } + + puppet_node { 'puppet_agent2': + role => 'pe_agent', + machine => "Cloudstack_instance[foo5]", + keypair => Cloudstack_keypair['dans_keypair4'], + require => [Cloudstack_keypair['dans_keypair4'], Puppet_node['puppet_master']], + puppetmaster => "Cloudstack_instance[foo3]", + classes => {'apache' => {}} + } +} From bf12fbda6333c94e25aef7b9fe205887fb4b9804 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Wed, 13 Mar 2013 10:32:19 -0700 Subject: [PATCH 12/15] remove commented out key pairs cleans up some commented out code, fixes formatting --- lib/puppet/provider/cloudstack_instance/default.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/puppet/provider/cloudstack_instance/default.rb b/lib/puppet/provider/cloudstack_instance/default.rb index f60750f..2f977ac 100644 --- a/lib/puppet/provider/cloudstack_instance/default.rb +++ b/lib/puppet/provider/cloudstack_instance/default.rb @@ -9,7 +9,6 @@ def self.instances # I may need to fail if the server does not have a name? connection.servers.collect do |server| - #require 'ruby-debug';debugger if (server.state != 'Destroyed') and (server.state != 'Stopping') Puppet.debug("Found #{server.display_name} in state: #{server.state}") Puppet.debug("Found #{server.display_name} in state: #{server.state}") @@ -31,7 +30,6 @@ def self.instances :group => server.group, # for some reason the keypair does not seem to be returned from listvminstnaces :keypair => server.key_pair, - #:keypair => server.keypair, :ensure => :present # I may want to print network information here ) @@ -61,9 +59,8 @@ def create :flavor_id => flavor_id, :zone_id => zone_id, :network_ids => network_id, - :key_pair => resource[:keypair], + :key_pair => resource[:keypair], :group => resource[:group] - #:keypair => resource[:keypair] ) @property_hash[:ensure] = :present @property_hash[:internal_ipaddress] = response.nics.first['ipaddress'] From 3bac5e1a805b6f1a2af14ae0a2322f0645952549 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Wed, 13 Mar 2013 10:54:39 -0700 Subject: [PATCH 13/15] allow key pairs to be wither references or strings so that you can refer to previously created key pairs. also, ensure that any specified key pairs are auto-required. --- lib/puppet/type/cloudstack_instance.rb | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/puppet/type/cloudstack_instance.rb b/lib/puppet/type/cloudstack_instance.rb index cd93e4e..badea8f 100644 --- a/lib/puppet/type/cloudstack_instance.rb +++ b/lib/puppet/type/cloudstack_instance.rb @@ -29,10 +29,6 @@ desc 'internal ip address. Puppet assumes that instances can only have one' end - newproperty(:keypair) do - desc 'keypair to associate with system' - end - newproperty(:flavor) do desc 'name of flavor' end @@ -82,7 +78,23 @@ def insync?(is) desc 'used to set userdata when an instance is created' end + newproperty(:keypair) do + desc 'keypair to associate with system' + munge do |value| + value = value.to_s + if value =~ /^Cloudstack_keypair\[(\S+)\]/ + $1 + elsif value =~ /^(\S+)\[(\S+)\]/ + fail("#{$1} is not a valid type, expected Cloudstack_keypair") + else + value + end + end + end + autorequire(:cloudstack_keypair) do + [self[:keypair]] + end # this causes querying the resources to fail ;( #validate do # unless self[:zone_id] and self[:template_id] and self[:service_offering_id] From 329daea9b61f3f562c65e15e955ff86e9217a5e1 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Wed, 13 Mar 2013 10:54:49 -0700 Subject: [PATCH 14/15] Add exists method to keypair --- lib/puppet/provider/cloudstack_keypair/default.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/puppet/provider/cloudstack_keypair/default.rb b/lib/puppet/provider/cloudstack_keypair/default.rb index 6a755ef..317229a 100644 --- a/lib/puppet/provider/cloudstack_keypair/default.rb +++ b/lib/puppet/provider/cloudstack_keypair/default.rb @@ -17,6 +17,10 @@ def self.instances end end + def exists? + @property_hash[:ensure] == :present + end + def create #domain_id = get_domain_id(resource[:domain]) #project_id = get_project_id(resource[:project]) From 042cd9d532299c3230fd05a50646628c5ed3b742 Mon Sep 17 00:00:00 2001 From: Dan Bode Date: Fri, 22 Mar 2013 17:32:56 -0700 Subject: [PATCH 15/15] more development on the ssh provider --- lib/puppet/provider/puppet_node/ssh.rb | 7 +++++-- lib/puppet/type/puppet_node.rb | 29 +++++++++++++++++++------- templates/fragments/options.erb | 13 +++++++++--- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/lib/puppet/provider/puppet_node/ssh.rb b/lib/puppet/provider/puppet_node/ssh.rb index 8877204..66b88dc 100644 --- a/lib/puppet/provider/puppet_node/ssh.rb +++ b/lib/puppet/provider/puppet_node/ssh.rb @@ -40,9 +40,11 @@ def run_remote_puppet_script(script_name) if script_name == 'pe_master' answers = 'https://raw.github.com/gist/4229037/d03453ae789797909745b63d44611adc5090c050/gistfile1.txt' + modulepath = '/etc/puppetlabs/puppet/modules' elsif script_name == 'pe_agent' answers = 'https://raw.github.com/gist/3299812/9dea06dba93d218c61e5fa9d9e928a265c137239/answers' else + modulepath = '/etc/puppet/modules' Puppet.debug("Script name: #{script_name} does not need an answers file") end @@ -59,7 +61,8 @@ def run_remote_puppet_script(script_name) 'answers_payload' => answers, 'certname' => machinehostname(resource[:machine]), 'puppetmaster' => machinehostname(resource[:puppetmaster]), - 'facts' => 'classes' => resource['classes'].to_pson + 'facts' => {'classes' => resource['classes'].to_pson}, + 'modulepath' => modulepath } Puppet.debug("Compiling script #{script_name} with options:\n#{compile_options.inspect}") @@ -93,7 +96,7 @@ def getprivatekey(key) if key =~ /(Cloudstack_keypair\[(\S+)?\])/ return model.catalog.resource($1).provider.key_file_path else - puts 'It is a path' + puts 'It is a path' return key end key diff --git a/lib/puppet/type/puppet_node.rb b/lib/puppet/type/puppet_node.rb index 995906f..255161e 100644 --- a/lib/puppet/type/puppet_node.rb +++ b/lib/puppet/type/puppet_node.rb @@ -10,18 +10,10 @@ desc 'classes to apply to the node. Only makes sense for an agent' end - newparam(:machine) do - desc 'machine where puppet actions are performed. Takes a reference or a hostname/ipaddress' - end - newproperty(:modules) do desc 'modules to install from the forge (only makes sense for a master role' end - newparam(:keypair) do - desc 'keypair to use to connect' - end - newproperty(:role) do desc 'role of this puppet instance' newvalues(:agent, :master, :pe_agent, :pe_master, :apply) @@ -48,6 +40,27 @@ def insync?(is) defaultto {} end + newparam(:keypair) do + desc 'keypair to use to connect' + validate do |value| + unless value.to_s =~ /^Cloudstack_keypair\[(\S+)\]/ + fail("#{value} is not a valid keypair reference, expected Cloudstack_keypair[]") + end + end + end + + newparam(:machine) do + desc 'machine where puppet actions are performed. Takes a reference or a hostname/ipaddress' + end + + autorequire(:cloudstack_keypair) do + if self[:keypair].to_s =~ /^Cloudstack_keypair\[(\S+)\]/ + [$1] + else + [] + end + end + autorequire(:cloudstack_instance) do instances = [] if self[:machine].to_s =~ /Cloudstack_instance\[(\S+?)\](\[\S+?\])?/ diff --git a/templates/fragments/options.erb b/templates/fragments/options.erb index 04c84c7..9c83a63 100644 --- a/templates/fragments/options.erb +++ b/templates/fragments/options.erb @@ -2,6 +2,13 @@ # install git and rubygems apt-get update || true + +# install epel +if [ -f /etc/redhat-release ]; then + puppet module install stahnma/epel + puppet apply -e 'include epel' +fi + apt-get install -y git-core || yum install -y git apt-get install -y rubygems || yum install -y rubygems @@ -34,12 +41,12 @@ return=$? #This fuction is not called if no custom facts are given # #The Puppet Labs' stdlib module is required for this to work -#It can be installed using: +#It can be installed using: # 'puppet-module install puppetlabs/stdlib' mkdir -p /etc/facter/facts.d -<% if options[:facts] %> +<% if options['facts'] %> echo "Installing custom facts" -<% options[:facts].each do |fact,value| %> +<% options['facts'].each do |fact,value| %> echo <%= fact %>=<%= value %> > /etc/facter/facts.d/<%= fact %>.txt <% end %> <% end %>