public class JavaActionExecutor extends ActionExecutor
ActionExecutor.Context, ActionExecutor.RETRYPOLICYACTION_RETRY_INTERVAL, ACTION_RETRY_POLICY, CONF_PREFIX, ERROR_OTHER, MAX_RETRIES, OOZIE_ACTION_YARN_TAG, RETRY_INTERVAL| Modifier | Constructor and Description |
|---|---|
|
JavaActionExecutor() |
protected |
JavaActionExecutor(String type) |
| Modifier and Type | Method and Description |
|---|---|
protected void |
addActionLibs(org.apache.hadoop.fs.Path appPath,
org.apache.hadoop.conf.Configuration conf) |
protected void |
addActionSpecificEnvVars(Map<String,String> env)
Adds action-specific environment variables.
|
protected void |
addAppNameContext(WorkflowAction action,
ActionExecutor.Context context) |
protected void |
addShareLib(org.apache.hadoop.conf.Configuration conf,
String[] actionShareLibNames) |
protected void |
addSystemShareLibForAction(org.apache.hadoop.conf.Configuration conf) |
void |
check(ActionExecutor.Context context,
WorkflowAction action)
Check if an action has completed.
|
org.apache.hadoop.conf.Configuration |
createBaseHadoopConf(ActionExecutor.Context context,
org.jdom.Element actionXml) |
protected org.apache.hadoop.conf.Configuration |
createBaseHadoopConf(ActionExecutor.Context context,
org.jdom.Element actionXml,
boolean loadResources) |
protected org.apache.hadoop.mapred.JobClient |
createJobClient(ActionExecutor.Context context,
org.apache.hadoop.conf.Configuration jobConf)
Create job client object
|
protected org.apache.hadoop.yarn.client.api.YarnClient |
createYarnClient(ActionExecutor.Context context,
org.apache.hadoop.conf.Configuration jobConf)
Create yarn client object
|
void |
end(ActionExecutor.Context context,
WorkflowAction action)
End an action after it has executed.
|
protected HashMap<String,CredentialsProperties> |
getActionCredentialsProperties(ActionExecutor.Context context,
WorkflowAction action) |
protected void |
getActionData(org.apache.hadoop.fs.FileSystem actionFs,
WorkflowAction action,
ActionExecutor.Context context)
Get the output data of an action.
|
protected String |
getActualExternalId(WorkflowAction action)
Useful for overriding in actions that do subsequent job runs
such as the MapReduce Action, where the launcher job is not the
actual job that then gets monitored.
|
protected String |
getAppName(ActionExecutor.Context context) |
protected boolean |
getCaptureOutput(WorkflowAction action) |
static List<Class<?>> |
getCommonLauncherClasses() |
protected CredentialsProperties |
getCredProperties(ActionExecutor.Context context,
String credName) |
protected String |
getDefaultShareLibName(org.jdom.Element actionXml)
Returns the default sharelib name for the action if any.
|
List<Class<?>> |
getLauncherClasses() |
protected String |
getLauncherMain(org.apache.hadoop.conf.Configuration launcherConf,
org.jdom.Element actionXml) |
static int |
getMaxExternalStatsSize()
Get the maximum allowed size of stats
|
protected static int |
getMaxOutputData(org.apache.hadoop.conf.Configuration actionConf) |
String[] |
getShareLibFilesForActionConf() |
protected String[] |
getShareLibNames(ActionExecutor.Context context,
org.jdom.Element actionXml,
org.apache.hadoop.conf.Configuration conf)
Return the sharelib names for the action.
|
protected static String |
getTrimmedEncodedPath(String path) |
void |
initActionType()
Invoked once at system initialization time.
|
protected void |
injectCallback(ActionExecutor.Context context,
org.apache.hadoop.conf.Configuration conf) |
boolean |
isCompleted(String externalStatus)
Return if the external status indicates that the action has completed.
|
void |
kill(ActionExecutor.Context context,
WorkflowAction action)
Kill an action.
|
protected org.apache.hadoop.conf.Configuration |
loadHadoopDefaultResources(ActionExecutor.Context context,
org.jdom.Element actionXml) |
static void |
parseJobXmlAndConfiguration(ActionExecutor.Context context,
org.jdom.Element element,
org.apache.hadoop.fs.Path appPath,
org.apache.hadoop.conf.Configuration conf) |
static void |
parseJobXmlAndConfiguration(ActionExecutor.Context context,
org.jdom.Element element,
org.apache.hadoop.fs.Path appPath,
org.apache.hadoop.conf.Configuration conf,
boolean isLauncher) |
void |
prepareActionDir(org.apache.hadoop.fs.FileSystem actionFs,
ActionExecutor.Context context) |
boolean |
requiresNameNodeJobTracker()
Returns true if this action type requires a NameNode and JobTracker.
|
protected void |
setActionCompletionData(ActionExecutor.Context context,
org.apache.hadoop.fs.FileSystem actionFs)
Sets some data for the action on completion
|
protected void |
setActionTokenProperties(org.apache.hadoop.conf.Configuration jobconf)
Subclasses may override this method in order to take additional actions required for obtaining credential token(s).
|
protected void |
setCredentialTokens(org.apache.hadoop.security.Credentials credentials,
org.apache.hadoop.conf.Configuration jobconf,
ActionExecutor.Context context,
WorkflowAction action,
Map<String,CredentialsProperties> credPropertiesMap) |
void |
setLibFilesArchives(ActionExecutor.Context context,
org.jdom.Element actionXml,
org.apache.hadoop.fs.Path appPath,
org.apache.hadoop.conf.Configuration conf) |
void |
start(ActionExecutor.Context context,
WorkflowAction action)
Start an action.
|
void |
submitLauncher(org.apache.hadoop.fs.FileSystem actionFs,
ActionExecutor.Context context,
WorkflowAction action) |
boolean |
supportsConfigurationJobXML()
Returns true if this action type supports a Configuration and JobXML.
|
convertException, disableInit, enableInit, getActionDir, getActionDirPath, getActionSignal, getActionYarnTag, getMaxRetries, getOozieConf, getOozieRuntimeDir, getOozieSystemId, getRetryInterval, getRetryPolicy, getType, registerError, resetInitInfo, setMaxRetries, setRetryInterval, setRetryPolicypublic static final String RUNNING
public static final String SUCCEEDED
public static final String KILLED
public static final String FAILED
public static final String FAILED_KILLED
public static final String HADOOP_YARN_RM
public static final String HADOOP_NAME_NODE
public static final String OOZIE_COMMON_LIBDIR
public static final String DEFAULT_LAUNCHER_VCORES
public static final String DEFAULT_LAUNCHER_MEMORY_MB
public static final String DEFAULT_LAUNCHER_PRIORITY
public static final String DEFAULT_LAUNCHER_QUEUE
public static final String DEFAULT_LAUNCHER_MAX_ATTEMPS
public static final String LAUNCER_MODIFY_ACL
public static final String LAUNCER_VIEW_ACL
public static final String MAPREDUCE_TO_CLASSPATH
public static final String OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY
public static final String MAX_EXTERNAL_STATS_SIZE
public static final String ACL_VIEW_JOB
public static final String ACL_MODIFY_JOB
public static final String HADOOP_YARN_TIMELINE_SERVICE_ENABLED
public static final String HADOOP_YARN_UBER_MODE
public static final String OOZIE_ACTION_LAUNCHER_PREFIX
public static final String HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART
public static final String HADOOP_MAP_MEMORY_MB
public static final String HADOOP_CHILD_JAVA_OPTS
public static final String HADOOP_MAP_JAVA_OPTS
public static final String HADOOP_REDUCE_JAVA_OPTS
public static final String HADOOP_CHILD_JAVA_ENV
public static final String HADOOP_MAP_JAVA_ENV
public static final String HADOOP_JOB_CLASSLOADER
public static final String HADOOP_USER_CLASSPATH_FIRST
public static final String OOZIE_CREDENTIALS_SKIP
public static final String YARN_AM_RESOURCE_MB
public static final String YARN_AM_COMMAND_OPTS
public static final String YARN_AM_ENV
public static final int YARN_MEMORY_MB_MIN
protected static final String HADOOP_USER
public XConfiguration workflowConf
public JavaActionExecutor()
protected JavaActionExecutor(String type)
public static List<Class<?>> getCommonLauncherClasses()
public List<Class<?>> getLauncherClasses()
public void initActionType()
ActionExecutorIt can be used to register error information for the expected exceptions. Exceptions should be register from subclasses to superclasses to ensure proper detection, same thing that it is done in a normal catch.
This method should invoke the ActionExecutor.registerError(java.lang.String, org.apache.oozie.action.ActionExecutorException.ErrorType, java.lang.String) method to register
all its possible errors.
Subclasses overriding must invoke super.
initActionType in class ActionExecutorpublic static int getMaxExternalStatsSize()
public org.apache.hadoop.conf.Configuration createBaseHadoopConf(ActionExecutor.Context context, org.jdom.Element actionXml)
protected org.apache.hadoop.conf.Configuration createBaseHadoopConf(ActionExecutor.Context context, org.jdom.Element actionXml, boolean loadResources)
protected org.apache.hadoop.conf.Configuration loadHadoopDefaultResources(ActionExecutor.Context context, org.jdom.Element actionXml)
public static void parseJobXmlAndConfiguration(ActionExecutor.Context context, org.jdom.Element element, org.apache.hadoop.fs.Path appPath, org.apache.hadoop.conf.Configuration conf) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException
public static void parseJobXmlAndConfiguration(ActionExecutor.Context context, org.jdom.Element element, org.apache.hadoop.fs.Path appPath, org.apache.hadoop.conf.Configuration conf, boolean isLauncher) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException
public void prepareActionDir(org.apache.hadoop.fs.FileSystem actionFs, ActionExecutor.Context context) throws ActionExecutorException
ActionExecutorExceptionprotected void addShareLib(org.apache.hadoop.conf.Configuration conf, String[] actionShareLibNames) throws ActionExecutorException
ActionExecutorExceptionprotected void addSystemShareLibForAction(org.apache.hadoop.conf.Configuration conf) throws ActionExecutorException
ActionExecutorExceptionprotected void addActionLibs(org.apache.hadoop.fs.Path appPath, org.apache.hadoop.conf.Configuration conf) throws ActionExecutorException
ActionExecutorExceptionpublic void setLibFilesArchives(ActionExecutor.Context context, org.jdom.Element actionXml, org.apache.hadoop.fs.Path appPath, org.apache.hadoop.conf.Configuration conf) throws ActionExecutorException
ActionExecutorExceptionprotected static String getTrimmedEncodedPath(String path)
protected String getLauncherMain(org.apache.hadoop.conf.Configuration launcherConf, org.jdom.Element actionXml)
protected static int getMaxOutputData(org.apache.hadoop.conf.Configuration actionConf)
protected void injectCallback(ActionExecutor.Context context, org.apache.hadoop.conf.Configuration conf)
public void submitLauncher(org.apache.hadoop.fs.FileSystem actionFs, ActionExecutor.Context context, WorkflowAction action) throws ActionExecutorException
ActionExecutorExceptionprotected void addAppNameContext(WorkflowAction action, ActionExecutor.Context context)
protected String getAppName(ActionExecutor.Context context)
protected void setCredentialTokens(org.apache.hadoop.security.Credentials credentials, org.apache.hadoop.conf.Configuration jobconf, ActionExecutor.Context context, WorkflowAction action, Map<String,CredentialsProperties> credPropertiesMap) throws Exception
Exceptionprotected void setActionTokenProperties(org.apache.hadoop.conf.Configuration jobconf)
jobconf - workflow action configurationprotected HashMap<String,CredentialsProperties> getActionCredentialsProperties(ActionExecutor.Context context, WorkflowAction action) throws Exception
Exceptionprotected CredentialsProperties getCredProperties(ActionExecutor.Context context, String credName) throws Exception
Exceptionpublic void start(ActionExecutor.Context context, WorkflowAction action) throws ActionExecutorException
ActionExecutor The ActionExecutor.Context.setStartData(java.lang.String, java.lang.String, java.lang.String) method must be called within this method.
If the
action has completed, the ActionExecutor.Context.setExecutionData(java.lang.String, java.util.Properties) method must be called within this method.
start in class ActionExecutorcontext - executor context.action - the action to start.ActionExecutorException - thrown if the action could not start.public void end(ActionExecutor.Context context, WorkflowAction action) throws ActionExecutorException
ActionExecutor The ActionExecutor.Context.setEndData(org.apache.oozie.client.WorkflowAction.Status, java.lang.String) method must be called within this
method.
end in class ActionExecutorcontext - executor context.action - the action to end.ActionExecutorException - thrown if the action could not end.protected org.apache.hadoop.mapred.JobClient createJobClient(ActionExecutor.Context context, org.apache.hadoop.conf.Configuration jobConf) throws HadoopAccessorException
context - jobConf - HadoopAccessorExceptionprotected org.apache.hadoop.yarn.client.api.YarnClient createYarnClient(ActionExecutor.Context context, org.apache.hadoop.conf.Configuration jobConf) throws HadoopAccessorException
context - jobConf - HadoopAccessorExceptionprotected String getActualExternalId(WorkflowAction action)
action - protected void addActionSpecificEnvVars(Map<String,String> env)
env - action specific environment variables stored as Map of key and value.public void check(ActionExecutor.Context context, WorkflowAction action) throws ActionExecutorException
ActionExecutor If the action
has completed, the ActionExecutor.Context.setExecutionData(java.lang.String, java.util.Properties) method must be called within this method.
If the action
has not completed, the ActionExecutor.Context.setExternalStatus(java.lang.String) method must be called within this method.
check in class ActionExecutorcontext - executor context.action - the action to end.ActionExecutorException - thrown if the action could not be checked.protected void getActionData(org.apache.hadoop.fs.FileSystem actionFs, WorkflowAction action, ActionExecutor.Context context) throws HadoopAccessorException, org.jdom.JDOMException, IOException, URISyntaxException
actionFs - the FileSystem objectaction - the Workflow actioncontext - executor contextHadoopAccessorExceptionorg.jdom.JDOMExceptionIOExceptionURISyntaxExceptionprotected boolean getCaptureOutput(WorkflowAction action) throws org.jdom.JDOMException
org.jdom.JDOMExceptionpublic void kill(ActionExecutor.Context context, WorkflowAction action) throws ActionExecutorException
ActionExecutor The ActionExecutor.Context.setEndData(org.apache.oozie.client.WorkflowAction.Status, java.lang.String) method must be called within this method.
kill in class ActionExecutorcontext - executor context.action - the action to kill.ActionExecutorException - thrown if the action could not be killed.public boolean isCompleted(String externalStatus)
ActionExecutorisCompleted in class ActionExecutorexternalStatus - external status to check.protected String[] getShareLibNames(ActionExecutor.Context context, org.jdom.Element actionXml, org.apache.hadoop.conf.Configuration conf)
SharelibResolver for details.context - executor context.actionXml - the action xml.conf - action configuration.protected String getDefaultShareLibName(org.jdom.Element actionXml)
actionXml - the action XML fragment.NULL if none.public String[] getShareLibFilesForActionConf()
protected void setActionCompletionData(ActionExecutor.Context context, org.apache.hadoop.fs.FileSystem actionFs) throws IOException, HadoopAccessorException, URISyntaxException
context - executor contextactionFs - the FileSystem objectIOExceptionHadoopAccessorExceptionURISyntaxExceptionpublic boolean requiresNameNodeJobTracker()
ActionExecutorrequiresNameNodeJobTracker in class ActionExecutorpublic boolean supportsConfigurationJobXML()
ActionExecutorsupportsConfigurationJobXML in class ActionExecutorCopyright © 2020 Apache Software Foundation. All rights reserved.