org.apache.hadoop.streaming
Class StreamJob

java.lang.Object
  org.apache.hadoop.streaming.StreamJob

public class StreamJob
extends Object

All the client-side work happens here. (Jar packaging, MapRed job submission and monitoring)

Field Summary

protected String	additionalConfSpec_
protected String	addTaskEnvironment_
protected URI[]	archiveURIs
protected String[]	argv_
protected String	cacheArchives
protected String	cacheFiles
protected String	cluster_
protected String	comCmd_
protected Configuration	config_
protected ArrayList	configPath_
protected int	debug_
protected boolean	detailedUsage_
protected Environment	env_
protected URI[]	fileURIs
protected String	hadoopAliasConf_
protected boolean	hasSimpleInputSpecs_
protected String	inputFormatSpec_
protected ArrayList	inputSpecs_
protected String	inReaderSpec_
protected String	jar_
protected JobClient	jc_
protected JobConf	jobConf_
protected JobID	jobId_
protected static String	LINK_URI
protected boolean	localHadoop_
protected static org.apache.commons.logging.Log	LOG
protected String	mapCmd_
protected String	mapDebugSpec_
protected boolean	mayExit_
protected long	minRecWrittenToEnableSkip_
protected String	numReduceTasksSpec_
protected String	output_
protected String	outputFormatSpec_
protected boolean	outputSingleNode_
protected ArrayList	packageFiles_
protected String	partitionerSpec_
protected String	redCmd_
protected String	reduceDebugSpec_
protected RunningJob	running_
protected TreeSet	seenPrimary_
protected ArrayList	shippedCanonFiles_
protected TreeMap<String,String>	userJobConfProps_
protected boolean	verbose_

Constructor Summary

StreamJob(String[] argv, boolean mayExit)

Method Summary

void	exitUsage(boolean detailed)
void	fail(String message)
protected String	getClusterNick()
protected String	getHadoopClientHome()
protected String	getJobTrackerHostPort()
protected void	getURIs(String lcacheArchives, String lcacheFiles) get the uris of all the files/caches
int	go() This is the method that actually intializes the job conf and submits the job to the jobtracker
protected void	init()
protected boolean	isLocalHadoop()
protected void	jobInfo()
protected void	listJobConfProperties() Prints out the jobconf properties on stdout when verbose is specified.
protected void	msg(String msg)
protected String	packageJobJar()
protected void	setJobConf()
protected void	setUserJobConfProps(boolean doEarlyProps) This method sets the user jobconf variable specified by user using -jobconf key=value
int	submitAndMonitorJob()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

protected static final org.apache.commons.logging.Log LOG

mayExit_

protected boolean mayExit_

argv_

protected String[] argv_

verbose_

protected boolean verbose_

detailedUsage_

protected boolean detailedUsage_

debug_

protected int debug_

env_

protected Environment env_

jar_

protected String jar_

localHadoop_

protected boolean localHadoop_

config_

protected Configuration config_

jobConf_

protected JobConf jobConf_

jc_

protected JobClient jc_

inputSpecs_

protected ArrayList inputSpecs_

seenPrimary_

protected TreeSet seenPrimary_

hasSimpleInputSpecs_

protected boolean hasSimpleInputSpecs_

packageFiles_

protected ArrayList packageFiles_

shippedCanonFiles_

protected ArrayList shippedCanonFiles_

userJobConfProps_

protected TreeMap<String,String> userJobConfProps_

output_

protected String output_

mapCmd_

protected String mapCmd_

comCmd_

protected String comCmd_

redCmd_

protected String redCmd_

cluster_

protected String cluster_

cacheFiles

protected String cacheFiles

cacheArchives

protected String cacheArchives

fileURIs

protected URI[] fileURIs

archiveURIs

protected URI[] archiveURIs

configPath_

protected ArrayList configPath_

hadoopAliasConf_

protected String hadoopAliasConf_

inReaderSpec_

protected String inReaderSpec_

inputFormatSpec_

protected String inputFormatSpec_

outputFormatSpec_

protected String outputFormatSpec_

partitionerSpec_

protected String partitionerSpec_

numReduceTasksSpec_

protected String numReduceTasksSpec_

additionalConfSpec_

protected String additionalConfSpec_

mapDebugSpec_

protected String mapDebugSpec_

reduceDebugSpec_

protected String reduceDebugSpec_

addTaskEnvironment_

protected String addTaskEnvironment_

outputSingleNode_

protected boolean outputSingleNode_

minRecWrittenToEnableSkip_

protected long minRecWrittenToEnableSkip_

running_

protected RunningJob running_

jobId_

protected JobID jobId_

LINK_URI

protected static String LINK_URI

Constructor Detail

StreamJob

public StreamJob(String[] argv,
                 boolean mayExit)

Method Detail

go

public int go()
       throws IOException

This is the method that actually intializes the job conf and submits the job to the jobtracker

Throws:

IOException

init

protected void init()

msg

protected void msg(String msg)

exitUsage

public void exitUsage(boolean detailed)

fail

public void fail(String message)

getHadoopClientHome

protected String getHadoopClientHome()

isLocalHadoop

protected boolean isLocalHadoop()

getClusterNick

protected String getClusterNick()

packageJobJar

protected String packageJobJar()
                        throws IOException

Returns:

path to the created Jar file or null if no files are necessary.

Throws:

IOException

setUserJobConfProps

protected void setUserJobConfProps(boolean doEarlyProps)

This method sets the user jobconf variable specified by user using -jobconf key=value

Parameters:

doEarlyProps -

getURIs

protected void getURIs(String lcacheArchives,
                       String lcacheFiles)

get the uris of all the files/caches

setJobConf

protected void setJobConf()
                   throws IOException

Throws:

IOException

listJobConfProperties

protected void listJobConfProperties()

Prints out the jobconf properties on stdout when verbose is specified.

getJobTrackerHostPort

protected String getJobTrackerHostPort()

jobInfo

protected void jobInfo()

submitAndMonitorJob

public int submitAndMonitorJob()
                        throws IOException

Throws:

IOException