(BKR-1132) adds concept of manual_test and manual_step

Adds an option --exec-manual-tests

adds manual_tests. This allows use to do some automated setup for manual
test cases, but also skips them when run in CI

adds manual_step. This allows us to pass or fail a test after prompting
the tester

lib/beaker/dsl/structure.rb

@@ -54,6 +54,63 @@ def step step_name, &block
         end
       end
 
+      # Provides a method to help manual tests. So we can use beaker to set up
+      # the environment, then prompt a user to manually check the setup.
+      # @param [String] step_name The name of the step to be logged.
+      def manual_step step_name
+        require 'readline'
+        logger.notify "\n* #{step_name}\n"
+        if(!@options.has_key?(:exec_manual_tests))
+          # if the option -exec-manual-tests is not set then this has executed outside of a manual tests
+          # so we raise an error to avoid issues
+          raise('--exec-manual-tests option not set, this means a manual_step was used outside a manual_test')
+        end
+
+        set_current_step_name(step_name)
+        # Here we prompt the user to tell us if the step passed or failed
+        loop do 
+          input = Readline.readline('Did this step pass, Y/n? ', true).squeeze(" ").strip.downcase
+          if %w(y yes).include?(input)
+            break
+          elsif %w(n no).include?(input)
+            # if the step failed, the user can enter a fail message.
+            # we loops to ensure they give use a fail message
+            fail_message = ''
+            loop do
+              fail_message = Readline.readline('What was the reason for failure? ', true).squeeze(" ").strip
+              if fail_message == ''
+                # if nothing is entered we tell the user to enter something
+                puts "No reason for failure given, please enter reason for failure."
+              else
+                break
+              end
+            end
+            raise Beaker::DSL::FailTest, fail_message
+          else
+            # if something other than Y or n is returned we ask them again
+            puts "Please enter Y or n."
+          end
+        end
+      end
+
+      # Provides a method to mark manual tests.
+      # If the --exec-manual-tests param is not set then we skip the test
+      # this is so manual tests do not execute by mistake
+      # @param [String] manual_test_name The name of the test to be logged.
+      # @param [Proc] block The actions to be performed during this test.
+      #
+      def manual_test manual_test_name, &block
+        if(@options.has_key?(:exec_manual_tests) && @options[:exec_manual_tests] == true)
+          # here the option is set so we run the test as normal
+          test_name manual_test_name, &block 
+        else
+          # here no option was set so we log the test name and skip it
+          test_name manual_test_name
+          raise( Beaker::DSL::SkipTest,
+                 '--exec-manual-tests option not set, so skipping manual test' )
+        end
+      end
+
       # Provides a method to name tests.
       #
       # @param [String] my_name The name of the test to be logged.

lib/beaker/options/command_line_parser.rb

@@ -94,6 +94,12 @@ def initialize
             @cmd_options[:debug_errors] = bool
           end
 
+          opts.on '--exec-manual-tests',
+                  'Execute manual tests',
+                  '(default: false)' do |bool|
+            @cmd_options[:exec_manual_tests] = bool
+          end
+
           opts.on '--root-keys',
                   'Install puppetlabs pubkeys for superuser',
                   '(default: false)' do |bool|

lib/beaker/subcommand.rb

@@ -52,6 +52,7 @@ def initialize(*args)
     class_option :'exclude-tags', :type => :string, :group => 'Beaker run'
     class_option :'xml-time-order', :type => :boolean, :group => 'Beaker run'
     class_option :'debug-errors', :type => :boolean, :group => 'Beaker run'
+    class_option :'exec_manual_tests', :type => :boolean, :group => 'Beaker run'
 
     # The following are listed as deprecated in beaker --help, but needed now for
     # feature parity for beaker 3.x.

spec/beaker/dsl/structure_spec.rb

@@ -48,6 +48,109 @@ class ClassMixedWithDSLStructure
     end
   end
 
+  describe '#manual_step' do
+    context 'without exec manual test option' do
+      let( :options ) { {} }
+      it 'throws an error' do
+        expect( Readline ).not_to receive( :readline )
+        expect { subject.manual_step 'blah' do; end }.to raise_error StandardError
+      end
+    end
+
+    context 'with exec manual test option' do
+      let( :options ) { {exec_manual_tests: nil} }
+      it 'requires a name' do
+        expect { subject.manual_step do; end }.to raise_error ArgumentError
+      end
+
+      it 'notifies the logger' do
+        subject.instance_variable_set(:@options, options)
+        allow( subject ).to receive( :set_current_step_name )
+        expect( subject ).to receive( :logger ).and_return( logger )
+        expect( logger ).to receive( :notify )
+        allow( Readline ).to receive( :readline ).and_return( 'Y')
+        subject.manual_step 'blah'
+      end
+    end
+
+    context 'with exec manual test option set to true' do
+      let( :options ) { {exec_manual_tests: true} }
+      it 'requires a name' do
+        expect { subject.manual_step do; end }.to raise_error ArgumentError
+      end
+
+      it 'pass when user enters Y' do
+        subject.instance_variable_set(:@options, options)
+        allow( subject ).to receive( :set_current_step_name )
+        allow( subject ).to receive( :logger ).and_return( logger )
+        allow( logger ).to receive( :notify )
+        expect( Readline ).to receive( :readline ).and_return( 'Y')
+        subject.manual_step 'blahblah'
+      end
+
+      it 'fails when user enters n and uses default error when no message is entered' do
+        subject.instance_variable_set(:@options, options)
+        allow( subject ).to receive( :set_current_step_name )
+        allow( subject ).to receive( :logger ).and_return( logger )
+        allow( logger ).to receive( :notify )
+        expect( Readline ).to receive( :readline ).and_return('n', 'step failed')
+        expect { subject.manual_step 'blah two' do; end }.to raise_error(Beaker::DSL::FailTest, 'step failed')
+      end
+    end
+  end
+
+  describe '#manual_test' do
+    context 'without exec manual test option' do
+      let( :options ) { {} }
+      it 'requires a name' do
+        expect { subject.manual_test do; end }.to raise_error ArgumentError
+      end
+
+      it 'raises a skip test' do
+        subject.instance_variable_set(:@options, options)
+        allow( subject ).to receive( :logger ).and_return( logger )
+        allow( logger ).to receive( :notify )
+        test_name = 'random test name'
+        expect { subject.manual_test test_name do; end }.to raise_error Beaker::DSL::SkipTest
+      end
+    end
+
+    context 'with exec manual test option' do
+      let( :options ) { {exec_manual_tests: true} }
+      it 'requires a name' do
+        expect { subject.manual_test do; end }.to raise_error ArgumentError
+      end
+
+      it 'notifies the logger' do
+        subject.instance_variable_set(:@options, options)
+        expect( subject ).to receive( :logger ).and_return( logger )
+        expect( logger ).to receive( :notify )
+        subject.manual_test 'blah blah'
+      end
+
+      it 'yields if a block is given' do
+        subject.instance_variable_set(:@options, options)
+        expect( subject ).to receive( :logger ).and_return( logger ).exactly(3).times
+        expect( logger ).to receive( :notify )
+        expect( logger ).to receive( :step_in )
+        expect( logger ).to receive( :step_out )
+        expect( subject ).to receive( :foo )
+        subject.manual_test 'blah' do
+          subject.foo
+        end
+      end
+
+      it 'sets the metadata' do
+        subject.instance_variable_set(:@options, options)
+        allow( subject ).to receive( :logger ).and_return( logger )
+        allow( logger ).to receive( :notify )
+        test_name = 'test is setting metadata yay!'
+        subject.manual_test test_name
+        expect( metadata[:case][:name] ).to be === test_name
+      end
+    end
+  end
+
   describe '#test_name' do
 
     it 'requires a name' do