diff --git a/bin/mandy-hadoop b/bin/mandy-hadoop index 0df787d..e85c94c 100755 --- a/bin/mandy-hadoop +++ b/bin/mandy-hadoop @@ -110,10 +110,9 @@ begin bootstrap_file = File.expand_path(File.join(File.dirname(__FILE__), '..', 'bootstrap.rb')) inputreader = job.input_format == :xml ? "StreamXmlRecordReader,begin=<#{job.input_format_options[:xml_tag]} ,end=" : nil - command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop*streaming*.jar \ + command = %($HADOOP_HOME/bin/hadoop jar /usr/lib/hadoop-0.20-mapreduce/contrib/streaming/hadoop*streaming*.jar \ #{options.libjars.nil? ? '' : "-libjars \"#{options.libjars}\""} \ #{jobconf}\ - -files "#{payload}","#{bootstrap_file}" \ -conf '#{config}' \ -inputformat '#{inputformat}' \ -outputformat '#{outputformat}' \ @@ -122,7 +121,9 @@ begin -mapper "ruby bootstrap.rb #{File.basename(payload)} map #{filename} '#{job.name}'" \ -reducer "ruby bootstrap.rb #{File.basename(payload)} reduce #{filename} '#{job.name}'" \ #{ cmdenv.nil? ? '' : "-cmdenv #{cmdenv}" }\ - -output "#{output}" 2>&1) + -output "#{output}" \ + -file "#{payload}" \ + -file "#{bootstrap_file}" 2>&1) result = [] IO.popen(command, 'r') do |subprocess| diff --git a/bin/mandy-mkdirp b/bin/mandy-mkdirp new file mode 100755 index 0000000..aa5a8c6 --- /dev/null +++ b/bin/mandy-mkdirp @@ -0,0 +1,32 @@ +#!/usr/bin/env ruby +require 'optparse' +require 'ostruct' + +require "rubygems" +require "mandy" + +HadoopConfiguration.check_home_and_version + +exec('mandy-mkdir -h') unless ARGV.size >= 2 + + +options = OpenStruct.new + +OptionParser.new do |opts| + opts.banner = "USAGE: mandy-mkdirp directory [options]" + + opts.on("-c", "--conf HADOOP_CONF", "Use this cluster xml config file.") do |config| + options.config = config + end + + opts.on_tail("-h", "--help", "Show this message") do + puts opts + exit + end +end.parse! + +dir = ARGV[0] +config = options.config || 'cluster.xml' + +`$HADOOP_HOME/bin/hadoop fs -conf #{config} -mkdir -p #{dir}` +exit($?.exitstatus) diff --git a/mandy.gemspec b/mandy.gemspec index d647d78..d3ca8bc 100644 --- a/mandy.gemspec +++ b/mandy.gemspec @@ -31,6 +31,7 @@ Gem::Specification.new do |s| mandy-mv mandy-cp mandy-mkdir + mandy-mkdirp mandy-exists mandy-install mandy-lsr @@ -52,6 +53,7 @@ Gem::Specification.new do |s| bin/mandy-mv bin/mandy-cp bin/mandy-mkdir + bin/mandy-mkdirp bin/mandy-exists bin/mandy-install bin/mandy-lsr