001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.action.hadoop; 020 021import com.google.common.annotations.VisibleForTesting; 022import com.google.common.base.Strings; 023import com.google.common.collect.ImmutableList; 024import com.google.common.io.Closeables; 025import com.google.common.primitives.Ints; 026 027import java.io.File; 028import java.io.FileNotFoundException; 029import java.io.IOException; 030import java.io.StringReader; 031import java.net.ConnectException; 032import java.net.URI; 033import java.net.URISyntaxException; 034import java.net.UnknownHostException; 035import java.nio.ByteBuffer; 036import java.text.MessageFormat; 037import java.util.ArrayList; 038import java.util.Arrays; 039import java.util.Collection; 040import java.util.HashMap; 041import java.util.HashSet; 042import java.util.Iterator; 043import java.util.LinkedHashMap; 044import java.util.List; 045import java.util.Map; 046import java.util.Map.Entry; 047 048import org.apache.commons.io.IOUtils; 049import org.apache.hadoop.conf.Configuration; 050import org.apache.hadoop.filecache.DistributedCache; 051import org.apache.hadoop.fs.FileStatus; 052import org.apache.hadoop.fs.FileSystem; 053import org.apache.hadoop.fs.Path; 054import org.apache.hadoop.io.DataOutputBuffer; 055import org.apache.hadoop.ipc.RemoteException; 056import org.apache.hadoop.mapred.JobClient; 057import org.apache.hadoop.mapred.TaskLog; 058import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; 059import org.apache.hadoop.mapreduce.v2.util.MRApps; 060import org.apache.hadoop.security.AccessControlException; 061import org.apache.hadoop.security.Credentials; 062import org.apache.hadoop.security.UserGroupInformation; 063import org.apache.hadoop.util.DiskChecker; 064import org.apache.hadoop.util.StringUtils; 065import org.apache.hadoop.yarn.api.ApplicationConstants; 066import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; 067import org.apache.hadoop.yarn.api.records.ApplicationId; 068import org.apache.hadoop.yarn.api.records.ApplicationReport; 069import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; 070import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; 071import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; 072import org.apache.hadoop.yarn.api.records.LocalResource; 073import org.apache.hadoop.yarn.api.records.Priority; 074import org.apache.hadoop.yarn.api.records.Resource; 075import org.apache.hadoop.yarn.api.records.YarnApplicationState; 076import org.apache.hadoop.yarn.client.api.YarnClient; 077import org.apache.hadoop.yarn.client.api.YarnClientApplication; 078import org.apache.hadoop.yarn.conf.YarnConfiguration; 079import org.apache.hadoop.yarn.util.Apps; 080import org.apache.hadoop.yarn.util.ConverterUtils; 081import org.apache.hadoop.yarn.util.Records; 082import org.apache.oozie.WorkflowActionBean; 083import org.apache.oozie.WorkflowJobBean; 084import org.apache.oozie.action.ActionExecutor; 085import org.apache.oozie.action.ActionExecutorException; 086import org.apache.oozie.client.OozieClient; 087import org.apache.oozie.client.WorkflowAction; 088import org.apache.oozie.command.coord.CoordActionStartXCommand; 089import org.apache.oozie.command.wf.WorkflowXCommand; 090import org.apache.oozie.service.ConfigurationService; 091import org.apache.oozie.service.HadoopAccessorException; 092import org.apache.oozie.service.HadoopAccessorService; 093import org.apache.oozie.service.Services; 094import org.apache.oozie.service.ShareLibService; 095import org.apache.oozie.service.URIHandlerService; 096import org.apache.oozie.service.WorkflowAppService; 097import org.apache.oozie.util.ClasspathUtils; 098import org.apache.oozie.util.ELEvaluationException; 099import org.apache.oozie.util.ELEvaluator; 100import org.apache.oozie.util.FSUtils; 101import org.apache.oozie.util.JobUtils; 102import org.apache.oozie.util.LogUtils; 103import org.apache.oozie.util.PropertiesUtils; 104import org.apache.oozie.util.XConfiguration; 105import org.apache.oozie.util.XLog; 106import org.apache.oozie.util.XmlUtils; 107import org.jdom.Element; 108import org.jdom.JDOMException; 109import org.jdom.Namespace; 110 111import java.util.Objects; 112import java.util.Properties; 113import java.util.Set; 114 115import com.google.common.base.Preconditions; 116import com.google.common.collect.ImmutableMap; 117 118 119public class JavaActionExecutor extends ActionExecutor { 120 public static final String RUNNING = "RUNNING"; 121 public static final String SUCCEEDED = "SUCCEEDED"; 122 public static final String KILLED = "KILLED"; 123 public static final String FAILED = "FAILED"; 124 public static final String FAILED_KILLED = "FAILED/KILLED"; 125 public static final String HADOOP_YARN_RM = "yarn.resourcemanager.address"; 126 public static final String HADOOP_NAME_NODE = "fs.default.name"; 127 public static final String OOZIE_COMMON_LIBDIR = "oozie"; 128 129 public static final String DEFAULT_LAUNCHER_VCORES = "oozie.launcher.default.vcores"; 130 public static final String DEFAULT_LAUNCHER_MEMORY_MB = "oozie.launcher.default.memory.mb"; 131 public static final String DEFAULT_LAUNCHER_PRIORITY = "oozie.launcher.default.priority"; 132 public static final String DEFAULT_LAUNCHER_QUEUE = "oozie.launcher.default.queue"; 133 public static final String DEFAULT_LAUNCHER_MAX_ATTEMPS = "oozie.launcher.default.max.attempts"; 134 public static final String LAUNCER_MODIFY_ACL = "oozie.launcher.modify.acl"; 135 public static final String LAUNCER_VIEW_ACL = "oozie.launcher.view.acl"; 136 137 public static final String MAPREDUCE_TO_CLASSPATH = "mapreduce.needed.for"; 138 public static final String OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY = ActionExecutor.CONF_PREFIX 139 + MAPREDUCE_TO_CLASSPATH; 140 141 public static final String MAX_EXTERNAL_STATS_SIZE = "oozie.external.stats.max.size"; 142 public static final String ACL_VIEW_JOB = "mapreduce.job.acl-view-job"; 143 public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; 144 public static final String HADOOP_YARN_TIMELINE_SERVICE_ENABLED = "yarn.timeline-service.enabled"; 145 public static final String HADOOP_YARN_UBER_MODE = "mapreduce.job.ubertask.enable"; 146 public static final String OOZIE_ACTION_LAUNCHER_PREFIX = ActionExecutor.CONF_PREFIX + "launcher."; 147 public static final String HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART = 148 OOZIE_ACTION_LAUNCHER_PREFIX + "am.restart.kill.childjobs"; 149 public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb"; 150 public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts"; 151 public static final String HADOOP_MAP_JAVA_OPTS = "mapreduce.map.java.opts"; 152 public static final String HADOOP_REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts"; 153 public static final String HADOOP_CHILD_JAVA_ENV = "mapred.child.env"; 154 public static final String HADOOP_MAP_JAVA_ENV = "mapreduce.map.env"; 155 public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader"; 156 public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first"; 157 public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip"; 158 public static final String YARN_AM_RESOURCE_MB = "yarn.app.mapreduce.am.resource.mb"; 159 public static final String YARN_AM_COMMAND_OPTS = "yarn.app.mapreduce.am.command-opts"; 160 public static final String YARN_AM_ENV = "yarn.app.mapreduce.am.env"; 161 public static final int YARN_MEMORY_MB_MIN = 512; 162 163 private static final String JAVA_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.JavaMain"; 164 private static final String HADOOP_JOB_NAME = "mapred.job.name"; 165 private static final Set<String> DISALLOWED_PROPERTIES = new HashSet<String>(); 166 private static final String OOZIE_ACTION_NAME = "oozie.action.name"; 167 private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for."; 168 169 private static int maxActionOutputLen; 170 private static int maxExternalStatsSize; 171 private static int maxFSGlobMax; 172 173 protected static final String HADOOP_USER = "user.name"; 174 175 protected XLog LOG = XLog.getLog(getClass()); 176 private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir="; 177 178 public XConfiguration workflowConf = null; 179 180 static { 181 DISALLOWED_PROPERTIES.add(HADOOP_USER); 182 DISALLOWED_PROPERTIES.add(HADOOP_NAME_NODE); 183 DISALLOWED_PROPERTIES.add(HADOOP_YARN_RM); 184 } 185 186 public JavaActionExecutor() { 187 this("java"); 188 } 189 190 protected JavaActionExecutor(String type) { 191 super(type); 192 } 193 194 public static List<Class<?>> getCommonLauncherClasses() { 195 List<Class<?>> classes = new ArrayList<Class<?>>(); 196 classes.add(LauncherMain.class); 197 classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher()); 198 classes.add(LauncherAM.class); 199 classes.add(LauncherAMCallbackNotifier.class); 200 return classes; 201 } 202 203 public List<Class<?>> getLauncherClasses() { 204 List<Class<?>> classes = new ArrayList<Class<?>>(); 205 try { 206 classes.add(Class.forName(JAVA_MAIN_CLASS_NAME)); 207 } 208 catch (ClassNotFoundException e) { 209 throw new RuntimeException("Class not found", e); 210 } 211 return classes; 212 } 213 214 @Override 215 public void initActionType() { 216 super.initActionType(); 217 maxActionOutputLen = ConfigurationService.getInt(LauncherAM.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA); 218 //Get the limit for the maximum allowed size of action stats 219 maxExternalStatsSize = ConfigurationService.getInt(JavaActionExecutor.MAX_EXTERNAL_STATS_SIZE); 220 maxExternalStatsSize = (maxExternalStatsSize == -1) ? Integer.MAX_VALUE : maxExternalStatsSize; 221 //Get the limit for the maximum number of globbed files/dirs for FS operation 222 maxFSGlobMax = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX); 223 224 registerError(UnknownHostException.class.getName(), ActionExecutorException.ErrorType.TRANSIENT, "JA001"); 225 registerError(AccessControlException.class.getName(), ActionExecutorException.ErrorType.NON_TRANSIENT, 226 "JA002"); 227 registerError(DiskChecker.DiskOutOfSpaceException.class.getName(), 228 ActionExecutorException.ErrorType.NON_TRANSIENT, "JA003"); 229 registerError(org.apache.hadoop.hdfs.protocol.QuotaExceededException.class.getName(), 230 ActionExecutorException.ErrorType.NON_TRANSIENT, "JA004"); 231 registerError(org.apache.hadoop.hdfs.server.namenode.SafeModeException.class.getName(), 232 ActionExecutorException.ErrorType.NON_TRANSIENT, "JA005"); 233 registerError(ConnectException.class.getName(), ActionExecutorException.ErrorType.TRANSIENT, " JA006"); 234 registerError(JDOMException.class.getName(), ActionExecutorException.ErrorType.ERROR, "JA007"); 235 registerError(FileNotFoundException.class.getName(), ActionExecutorException.ErrorType.ERROR, "JA008"); 236 registerError(IOException.class.getName(), ActionExecutorException.ErrorType.TRANSIENT, "JA009"); 237 } 238 239 240 /** 241 * Get the maximum allowed size of stats 242 * 243 * @return maximum size of stats 244 */ 245 public static int getMaxExternalStatsSize() { 246 return maxExternalStatsSize; 247 } 248 249 static void checkForDisallowedProps(Configuration conf, String confName) throws ActionExecutorException { 250 for (String prop : DISALLOWED_PROPERTIES) { 251 if (conf.get(prop) != null) { 252 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA010", 253 "Property [{0}] not allowed in action [{1}] configuration", prop, confName); 254 } 255 } 256 } 257 258 public Configuration createBaseHadoopConf(Context context, Element actionXml) { 259 return createBaseHadoopConf(context, actionXml, true); 260 } 261 262 protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) { 263 264 Namespace ns = actionXml.getNamespace(); 265 String resourceManager; 266 final Element resourceManagerTag = actionXml.getChild("resource-manager", ns); 267 if (resourceManagerTag != null) { 268 resourceManager = resourceManagerTag.getTextTrim(); 269 } 270 else { 271 resourceManager = actionXml.getChild("job-tracker", ns).getTextTrim(); 272 } 273 274 String nameNode = actionXml.getChild("name-node", ns).getTextTrim(); 275 Configuration conf = null; 276 if (loadResources) { 277 conf = Services.get().get(HadoopAccessorService.class).createConfiguration(resourceManager); 278 } 279 else { 280 conf = new Configuration(false); 281 } 282 283 conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER)); 284 conf.set(HADOOP_YARN_RM, resourceManager); 285 conf.set(HADOOP_NAME_NODE, nameNode); 286 conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "true"); 287 288 // FIXME - think about this! 289 Element e = actionXml.getChild("config-class", ns); 290 if (e != null) { 291 conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim()); 292 } 293 294 return conf; 295 } 296 297 protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { 298 return createBaseHadoopConf(context, actionXml); 299 } 300 301 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) 302 throws ActionExecutorException { 303 try { 304 Namespace ns = actionXml.getNamespace(); 305 XConfiguration launcherConf = new XConfiguration(); 306 // Inject action defaults for launcher 307 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); 308 XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_YARN_RM), getType()); 309 310 new LauncherConfigurationInjector(actionDefaultConf).inject(launcherConf); 311 312 // Inject <job-xml> and <configuration> for launcher 313 try { 314 parseJobXmlAndConfiguration(context, actionXml, appPath, launcherConf, true); 315 } catch (HadoopAccessorException ex) { 316 throw convertException(ex); 317 } catch (URISyntaxException ex) { 318 throw convertException(ex); 319 } 320 XConfiguration.copy(launcherConf, conf); 321 // Inject config-class for launcher to use for action 322 Element e = actionXml.getChild("config-class", ns); 323 if (e != null) { 324 conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim()); 325 } 326 checkForDisallowedProps(launcherConf, "launcher configuration"); 327 return conf; 328 } 329 catch (IOException ex) { 330 throw convertException(ex); 331 } 332 } 333 334 void injectLauncherTimelineServiceEnabled(Configuration launcherConf, Configuration actionConf) { 335 // Getting delegation token for ATS. If tez-site.xml is present in distributed cache, turn on timeline service. 336 if (actionConf.get("oozie.launcher." + HADOOP_YARN_TIMELINE_SERVICE_ENABLED) == null 337 && ConfigurationService.getBoolean(OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_YARN_TIMELINE_SERVICE_ENABLED)) { 338 String cacheFiles = launcherConf.get("mapred.cache.files"); 339 if (cacheFiles != null && cacheFiles.contains("tez-site.xml")) { 340 launcherConf.setBoolean(HADOOP_YARN_TIMELINE_SERVICE_ENABLED, true); 341 } 342 } 343 } 344 345 public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf) 346 throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { 347 parseJobXmlAndConfiguration(context, element, appPath, conf, false); 348 } 349 350 public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf, 351 boolean isLauncher) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException { 352 Namespace ns = element.getNamespace(); 353 @SuppressWarnings("unchecked") 354 Iterator<Element> it = element.getChildren("job-xml", ns).iterator(); 355 HashMap<String, FileSystem> filesystemsMap = new HashMap<String, FileSystem>(); 356 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); 357 while (it.hasNext()) { 358 Element e = it.next(); 359 String jobXml = e.getTextTrim(); 360 Path pathSpecified = new Path(jobXml); 361 Path path = pathSpecified.isAbsolute() ? pathSpecified : new Path(appPath, jobXml); 362 FileSystem fs; 363 if (filesystemsMap.containsKey(path.toUri().getAuthority())) { 364 fs = filesystemsMap.get(path.toUri().getAuthority()); 365 } 366 else { 367 if (path.toUri().getAuthority() != null) { 368 fs = has.createFileSystem(context.getWorkflow().getUser(), path.toUri(), 369 has.createConfiguration(path.toUri().getAuthority())); 370 } 371 else { 372 fs = context.getAppFileSystem(); 373 } 374 filesystemsMap.put(path.toUri().getAuthority(), fs); 375 } 376 Configuration jobXmlConf = new XConfiguration(fs.open(path)); 377 try { 378 String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString(); 379 jobXmlConfString = XmlUtils.removeComments(jobXmlConfString); 380 jobXmlConfString = context.getELEvaluator().evaluate(jobXmlConfString, String.class); 381 jobXmlConf = new XConfiguration(new StringReader(jobXmlConfString)); 382 } 383 catch (ELEvaluationException ex) { 384 throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, "EL_EVAL_ERROR", ex 385 .getMessage(), ex); 386 } 387 catch (Exception ex) { 388 context.setErrorInfo("EL_ERROR", ex.getMessage()); 389 } 390 checkForDisallowedProps(jobXmlConf, "job-xml"); 391 if (isLauncher) { 392 new LauncherConfigurationInjector(jobXmlConf).inject(conf); 393 } else { 394 XConfiguration.copy(jobXmlConf, conf); 395 } 396 } 397 Element e = element.getChild("configuration", ns); 398 if (e != null) { 399 String strConf = XmlUtils.prettyPrint(e).toString(); 400 XConfiguration inlineConf = new XConfiguration(new StringReader(strConf)); 401 checkForDisallowedProps(inlineConf, "inline configuration"); 402 if (isLauncher) { 403 new LauncherConfigurationInjector(inlineConf).inject(conf); 404 } else { 405 XConfiguration.copy(inlineConf, conf); 406 } 407 } 408 } 409 410 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) 411 throws ActionExecutorException { 412 try { 413 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); 414 XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_YARN_RM), getType()); 415 XConfiguration.copy(actionDefaults, actionConf); 416 has.checkSupportedFilesystem(appPath.toUri()); 417 418 // Set the Java Main Class for the Java action to give to the Java launcher 419 setJavaMain(actionConf, actionXml); 420 421 parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf); 422 423 // set cancel.delegation.token in actionConf that child job doesn't cancel delegation token 424 actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false); 425 setRootLoggerLevel(actionConf); 426 return actionConf; 427 } 428 catch (IOException ex) { 429 throw convertException(ex); 430 } 431 catch (HadoopAccessorException ex) { 432 throw convertException(ex); 433 } 434 catch (URISyntaxException ex) { 435 throw convertException(ex); 436 } 437 } 438 439 /** 440 * Set root log level property in actionConf 441 * @param actionConf 442 */ 443 void setRootLoggerLevel(Configuration actionConf) { 444 String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherAMUtils.ROOT_LOGGER_LEVEL; 445 String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL; 446 447 // check if root log level has already mentioned in action configuration 448 String rootLogLevel = actionConf.get(oozieActionTypeRootLogger, actionConf.get(oozieActionRootLogger)); 449 if (rootLogLevel != null) { 450 // root log level is mentioned in action configuration 451 return; 452 } 453 454 // set the root log level which is mentioned in oozie default 455 rootLogLevel = ConfigurationService.get(oozieActionTypeRootLogger); 456 if (rootLogLevel != null && rootLogLevel.length() > 0) { 457 actionConf.set(oozieActionRootLogger, rootLogLevel); 458 } 459 else { 460 rootLogLevel = ConfigurationService.get(oozieActionRootLogger); 461 if (rootLogLevel != null && rootLogLevel.length() > 0) { 462 actionConf.set(oozieActionRootLogger, rootLogLevel); 463 } 464 } 465 } 466 467 Configuration addToCache(Configuration conf, Path appPath, String filePath, boolean archive) 468 throws ActionExecutorException { 469 470 URI uri = null; 471 try { 472 uri = new URI(getTrimmedEncodedPath(filePath)); 473 URI baseUri = appPath.toUri(); 474 if (uri.getScheme() == null) { 475 String resolvedPath = uri.getPath(); 476 if (!resolvedPath.startsWith("/")) { 477 resolvedPath = baseUri.getPath() + "/" + resolvedPath; 478 } 479 uri = new URI(baseUri.getScheme(), baseUri.getAuthority(), resolvedPath, uri.getQuery(), uri.getFragment()); 480 } 481 if (archive) { 482 DistributedCache.addCacheArchive(uri.normalize(), conf); 483 } 484 else { 485 String fileName = filePath.substring(filePath.lastIndexOf("/") + 1); 486 if (fileName.endsWith(".so") || fileName.contains(".so.")) { // .so files 487 uri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), fileName); 488 DistributedCache.addCacheFile(uri.normalize(), conf); 489 } 490 else if (fileName.endsWith(".jar")) { // .jar files 491 if (!fileName.contains("#")) { 492 String user = conf.get("user.name"); 493 494 if (FSUtils.isNotLocalFile(fileName)) { 495 Path pathToAdd = new Path(uri.normalize()); 496 Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf); 497 } 498 } 499 else { 500 DistributedCache.addCacheFile(uri.normalize(), conf); 501 } 502 } 503 else { // regular files 504 if (!fileName.contains("#")) { 505 uri = new URI(uri.getScheme(), uri.getAuthority(), uri.getPath(), uri.getQuery(), fileName); 506 } 507 DistributedCache.addCacheFile(uri.normalize(), conf); 508 } 509 } 510 DistributedCache.createSymlink(conf); 511 return conf; 512 } 513 catch (Exception ex) { 514 LOG.debug("Errors when add to DistributedCache. Path=" + 515 Objects.toString(uri, "<null>") + ", archive=" + archive + ", conf=" + 516 XmlUtils.prettyPrint(conf).toString()); 517 throw convertException(ex); 518 } 519 } 520 521 public void prepareActionDir(FileSystem actionFs, Context context) throws ActionExecutorException { 522 try { 523 Path actionDir = context.getActionDir(); 524 Path tempActionDir = new Path(actionDir.getParent(), actionDir.getName() + ".tmp"); 525 if (!actionFs.exists(actionDir)) { 526 try { 527 actionFs.mkdirs(tempActionDir); 528 actionFs.rename(tempActionDir, actionDir); 529 } 530 catch (IOException ex) { 531 actionFs.delete(tempActionDir, true); 532 actionFs.delete(actionDir, true); 533 throw ex; 534 } 535 } 536 } 537 catch (Exception ex) { 538 throw convertException(ex); 539 } 540 } 541 542 void cleanUpActionDir(FileSystem actionFs, Context context) throws ActionExecutorException { 543 try { 544 Path actionDir = context.getActionDir(); 545 if (!context.getProtoActionConf().getBoolean(WorkflowXCommand.KEEP_WF_ACTION_DIR, false) 546 && actionFs.exists(actionDir)) { 547 actionFs.delete(actionDir, true); 548 } 549 } 550 catch (Exception ex) { 551 throw convertException(ex); 552 } 553 } 554 555 protected void addShareLib(Configuration conf, String[] actionShareLibNames) 556 throws ActionExecutorException { 557 Set<String> confSet = new HashSet<String>(Arrays.asList(getShareLibFilesForActionConf() == null ? new String[0] 558 : getShareLibFilesForActionConf())); 559 560 Set<Path> sharelibList = new HashSet<Path>(); 561 562 if (actionShareLibNames != null) { 563 try { 564 ShareLibService shareLibService = Services.get().get(ShareLibService.class); 565 FileSystem fs = shareLibService.getFileSystem(); 566 if (fs != null) { 567 for (String actionShareLibName : actionShareLibNames) { 568 List<Path> listOfPaths = shareLibService.getShareLibJars(actionShareLibName); 569 if (listOfPaths != null && !listOfPaths.isEmpty()) { 570 for (Path actionLibPath : listOfPaths) { 571 String fragmentName = new URI(actionLibPath.toString()).getFragment(); 572 String fileName = fragmentName == null ? actionLibPath.getName() : fragmentName; 573 if (confSet.contains(fileName)) { 574 Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName, 575 actionLibPath); 576 if (jobXmlConf != null) { 577 checkForDisallowedProps(jobXmlConf, actionLibPath.getName()); 578 XConfiguration.injectDefaults(jobXmlConf, conf); 579 LOG.trace("Adding properties of " + actionLibPath + " to job conf"); 580 } 581 } 582 else { 583 // Filtering out duplicate jars or files 584 sharelibList.add(new Path(actionLibPath.toUri()) { 585 @Override 586 public int hashCode() { 587 return getName().hashCode(); 588 } 589 @Override 590 public String getName() { 591 try { 592 return (new URI(toString())).getFragment() == null ? new Path(toUri()).getName() 593 : (new URI(toString())).getFragment(); 594 } 595 catch (URISyntaxException e) { 596 throw new RuntimeException(e); 597 } 598 } 599 @Override 600 public boolean equals(Object input) { 601 if (input == null) { 602 return false; 603 } 604 if (input == this) { 605 return true; 606 } 607 if (!(input instanceof Path)) { 608 return false; 609 } 610 return getName().equals(((Path) input).getName()); 611 } 612 }); 613 } 614 } 615 } 616 } 617 } 618 addLibPathsToCache(conf, sharelibList); 619 } 620 catch (URISyntaxException ex) { 621 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "Error configuring sharelib", 622 ex.getMessage()); 623 } 624 catch (IOException ex) { 625 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", 626 ex.getMessage()); 627 } 628 } 629 } 630 631 protected void addSystemShareLibForAction(Configuration conf) throws ActionExecutorException { 632 ShareLibService shareLibService = Services.get().get(ShareLibService.class); 633 // ShareLibService is null for test cases 634 if (shareLibService != null) { 635 try { 636 List<Path> listOfPaths = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR); 637 if (listOfPaths.isEmpty()) { 638 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "EJ001", 639 "Could not locate Oozie sharelib"); 640 } 641 addLibPathsToClassPath(conf, listOfPaths); 642 addLibPathsToClassPath(conf, shareLibService.getSystemLibJars(getType())); 643 } 644 catch (IOException ex) { 645 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", 646 ex.getMessage()); 647 } 648 } 649 } 650 651 private void addLibPathsToClassPath(Configuration conf, List<Path> listOfPaths) 652 throws IOException, ActionExecutorException { 653 addLibPathsToClassPathOrCache(conf, listOfPaths, false); 654 } 655 656 private void addLibPathsToCache(Configuration conf, Set<Path> sharelibList) 657 throws IOException, ActionExecutorException { 658 addLibPathsToClassPathOrCache(conf, sharelibList, true); 659 } 660 661 662 private void addLibPathsToClassPathOrCache(Configuration conf, Collection<Path> sharelibList, boolean isAddToCache) 663 throws IOException, ActionExecutorException { 664 665 for (Path libPath : sharelibList) { 666 if (FSUtils.isLocalFile(libPath.toString())) { 667 conf = ClasspathUtils.addToClasspathFromLocalShareLib(conf, libPath); 668 } 669 else { 670 if (isAddToCache) { 671 addToCache(conf, libPath, libPath.toUri().getPath(), false); 672 } 673 else { 674 FileSystem fs = libPath.getFileSystem(conf); 675 JobUtils.addFileToClassPath(libPath, conf, fs); 676 DistributedCache.createSymlink(conf); 677 } 678 } 679 } 680 } 681 682 protected void addActionLibs(Path appPath, Configuration conf) throws ActionExecutorException { 683 String[] actionLibsStrArr = conf.getStrings("oozie.launcher.oozie.libpath"); 684 if (actionLibsStrArr != null) { 685 try { 686 for (String actionLibsStr : actionLibsStrArr) { 687 actionLibsStr = actionLibsStr.trim(); 688 if (actionLibsStr.length() > 0) 689 { 690 Path actionLibsPath = new Path(actionLibsStr); 691 String user = conf.get("user.name"); 692 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, 693 appPath.toUri(), conf); 694 if (fs.exists(actionLibsPath)) { 695 FileStatus[] files = fs.listStatus(actionLibsPath); 696 for (FileStatus file : files) { 697 addToCache(conf, appPath, file.getPath().toUri().getPath(), false); 698 } 699 } 700 } 701 } 702 } 703 catch (HadoopAccessorException ex){ 704 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, 705 ex.getErrorCode().toString(), ex.getMessage()); 706 } 707 catch (IOException ex){ 708 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, 709 "It should never happen", ex.getMessage()); 710 } 711 } 712 } 713 714 @SuppressWarnings("unchecked") 715 public void setLibFilesArchives(Context context, Element actionXml, Path appPath, Configuration conf) 716 throws ActionExecutorException { 717 Configuration proto = context.getProtoActionConf(); 718 719 // Workflow lib/ 720 String[] paths = proto.getStrings(WorkflowAppService.APP_LIB_PATH_LIST); 721 if (paths != null) { 722 for (String path : paths) { 723 addToCache(conf, appPath, path, false); 724 } 725 } 726 727 // Action libs 728 addActionLibs(appPath, conf); 729 730 // files and archives defined in the action 731 for (Element eProp : (List<Element>) actionXml.getChildren()) { 732 if (eProp.getName().equals("file")) { 733 String[] filePaths = eProp.getTextTrim().split(","); 734 for (String path : filePaths) { 735 addToCache(conf, appPath, path, false); 736 } 737 } 738 else if (eProp.getName().equals("archive")) { 739 String[] archivePaths = eProp.getTextTrim().split(","); 740 for (String path : archivePaths){ 741 addToCache(conf, appPath, path.trim(), true); 742 } 743 } 744 } 745 746 addAllShareLibs(appPath, conf, context, actionXml); 747 } 748 749 @VisibleForTesting 750 protected static String getTrimmedEncodedPath(String path) { 751 return path.trim().replace(" ", "%20"); 752 } 753 754 // Adds action specific share libs and common share libs 755 private void addAllShareLibs(Path appPath, Configuration conf, Context context, Element actionXml) 756 throws ActionExecutorException { 757 // Add action specific share libs 758 addActionShareLib(appPath, conf, context, actionXml); 759 // Add common sharelibs for Oozie and launcher jars 760 addSystemShareLibForAction(conf); 761 } 762 763 private void addActionShareLib(Path appPath, Configuration conf, Context context, Element actionXml) 764 throws ActionExecutorException { 765 XConfiguration wfJobConf = null; 766 try { 767 wfJobConf = getWorkflowConf(context); 768 } 769 catch (IOException ioe) { 770 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen", 771 ioe.getMessage()); 772 } 773 // Action sharelibs are only added if user has specified to use system libpath 774 if (conf.get(OozieClient.USE_SYSTEM_LIBPATH) == null) { 775 if (wfJobConf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, 776 ConfigurationService.getBoolean(OozieClient.USE_SYSTEM_LIBPATH))) { 777 // add action specific sharelibs 778 addShareLib(conf, getShareLibNames(context, actionXml, conf)); 779 } 780 } 781 else { 782 if (conf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, false)) { 783 // add action specific sharelibs 784 addShareLib(conf, getShareLibNames(context, actionXml, conf)); 785 } 786 } 787 } 788 789 790 protected String getLauncherMain(Configuration launcherConf, Element actionXml) { 791 return launcherConf.get(LauncherAM.CONF_OOZIE_ACTION_MAIN_CLASS, JavaMain.class.getName()); 792 } 793 794 private void setJavaMain(Configuration actionConf, Element actionXml) { 795 Namespace ns = actionXml.getNamespace(); 796 Element e = actionXml.getChild("main-class", ns); 797 if (e != null) { 798 actionConf.set(JavaMain.JAVA_MAIN_CLASS, e.getTextTrim()); 799 } 800 } 801 802 private static final String QUEUE_NAME = "mapred.job.queue.name"; 803 804 private static final Set<String> SPECIAL_PROPERTIES = new HashSet<String>(); 805 806 static { 807 SPECIAL_PROPERTIES.add(QUEUE_NAME); 808 SPECIAL_PROPERTIES.add(ACL_VIEW_JOB); 809 SPECIAL_PROPERTIES.add(ACL_MODIFY_JOB); 810 } 811 812 @SuppressWarnings("unchecked") 813 Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, 814 Configuration actionConf) throws ActionExecutorException { 815 try { 816 817 // app path could be a file 818 Path appPathRoot = new Path(context.getWorkflow().getAppPath()); 819 if (actionFs.isFile(appPathRoot)) { 820 appPathRoot = appPathRoot.getParent(); 821 } 822 823 // launcher job configuration 824 Configuration launcherJobConf = createBaseHadoopConf(context, actionXml); 825 // cancel delegation token on a launcher job which stays alive till child job(s) finishes 826 // otherwise (in mapred action), doesn't cancel not to disturb running child job 827 launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); 828 setupLauncherConf(launcherJobConf, actionXml, appPathRoot, context); 829 830 // Properties for when a launcher job's AM gets restarted 831 if (ConfigurationService.getBoolean(HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART)) { 832 // launcher time filter is required to prune the search of launcher tag. 833 // Setting coordinator action nominal time as launcher time as it child job cannot launch before nominal 834 // time. Workflow created time is good enough when workflow is running independently or workflow is 835 // rerunning from failed node. 836 long launcherTime = System.currentTimeMillis(); 837 String coordActionNominalTime = context.getProtoActionConf().get( 838 CoordActionStartXCommand.OOZIE_COORD_ACTION_NOMINAL_TIME); 839 if (coordActionNominalTime != null) { 840 launcherTime = Long.parseLong(coordActionNominalTime); 841 } 842 else if (context.getWorkflow().getCreatedTime() != null) { 843 launcherTime = context.getWorkflow().getCreatedTime().getTime(); 844 } 845 String actionYarnTag = getActionYarnTag(getWorkflowConf(context), context.getWorkflow(), action); 846 LauncherHelper.setupYarnRestartHandling(launcherJobConf, actionConf, actionYarnTag, launcherTime); 847 } 848 else { 849 LOG.info(MessageFormat.format("{0} is set to false, not setting YARN restart properties", 850 HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART)); 851 } 852 853 String actionShareLibProperty = actionConf.get(ACTION_SHARELIB_FOR + getType()); 854 if (actionShareLibProperty != null) { 855 launcherJobConf.set(ACTION_SHARELIB_FOR + getType(), actionShareLibProperty); 856 } 857 setLibFilesArchives(context, actionXml, appPathRoot, launcherJobConf); 858 859 // Inject Oozie job information if enabled. 860 injectJobInfo(launcherJobConf, actionConf, context, action); 861 862 injectLauncherCallback(context, launcherJobConf); 863 864 String jobId = context.getWorkflow().getId(); 865 String actionId = action.getId(); 866 Path actionDir = context.getActionDir(); 867 String recoveryId = context.getRecoveryId(); 868 869 // Getting the prepare XML from the action XML 870 Namespace ns = actionXml.getNamespace(); 871 Element prepareElement = actionXml.getChild("prepare", ns); 872 String prepareXML = ""; 873 if (prepareElement != null) { 874 if (prepareElement.getChildren().size() > 0) { 875 prepareXML = XmlUtils.prettyPrint(prepareElement).toString().trim(); 876 } 877 } 878 LauncherHelper.setupLauncherInfo(launcherJobConf, jobId, actionId, actionDir, recoveryId, actionConf, 879 prepareXML); 880 881 // Set the launcher Main Class 882 LauncherHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml)); 883 LauncherHelper.setupLauncherURIHandlerConf(launcherJobConf); 884 LauncherHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf)); 885 LauncherHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize); 886 LauncherHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax); 887 888 List<Element> list = actionXml.getChildren("arg", ns); 889 String[] args = new String[list.size()]; 890 for (int i = 0; i < list.size(); i++) { 891 args[i] = list.get(i).getTextTrim(); 892 } 893 LauncherHelper.setupMainArguments(launcherJobConf, args); 894 // backward compatibility flag - see OOZIE-2872 895 boolean nullArgsAllowed = ConfigurationService.getBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED); 896 launcherJobConf.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed); 897 898 // Make mapred.child.java.opts and mapreduce.map.java.opts equal, but give values from the latter priority; also append 899 // <java-opt> and <java-opts> and give those highest priority 900 StringBuilder opts = new StringBuilder(launcherJobConf.get(HADOOP_CHILD_JAVA_OPTS, "")); 901 if (launcherJobConf.get(HADOOP_MAP_JAVA_OPTS) != null) { 902 opts.append(" ").append(launcherJobConf.get(HADOOP_MAP_JAVA_OPTS)); 903 } 904 905 List<Element> javaopts = actionXml.getChildren("java-opt", ns); 906 907 // Either one or more <java-opt> element or one <java-opts> can be present since oozie-workflow-0.4 908 if (!javaopts.isEmpty()) { 909 for (Element opt : javaopts) { 910 opts.append(" ").append(opt.getTextTrim()); 911 } 912 } 913 else { 914 Element opt = actionXml.getChild("java-opts", ns); 915 if (opt != null) { 916 opts.append(" ").append(opt.getTextTrim()); 917 } 918 } 919 launcherJobConf.set(HADOOP_CHILD_JAVA_OPTS, opts.toString().trim()); 920 launcherJobConf.set(HADOOP_MAP_JAVA_OPTS, opts.toString().trim()); 921 922 injectLauncherTimelineServiceEnabled(launcherJobConf, actionConf); 923 924 // properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs) 925 // maybe we should add queue to the WF schema, below job-tracker 926 actionConfToLauncherConf(actionConf, launcherJobConf); 927 928 return launcherJobConf; 929 } 930 catch (Exception ex) { 931 throw convertException(ex); 932 } 933 } 934 935 @VisibleForTesting 936 protected static int getMaxOutputData(Configuration actionConf) { 937 String userMaxActionOutputLen = actionConf.get("oozie.action.max.output.data"); 938 if (userMaxActionOutputLen != null) { 939 Integer i = Ints.tryParse(userMaxActionOutputLen); 940 return i != null ? i : maxActionOutputLen; 941 } 942 return maxActionOutputLen; 943 } 944 945 protected void injectCallback(Context context, Configuration conf) { 946 String callback = context.getCallbackUrl(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN); 947 conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, callback); 948 } 949 950 void injectActionCallback(Context context, Configuration actionConf) { 951 // action callback needs to be injected only for mapreduce actions. 952 } 953 954 void injectLauncherCallback(Context context, Configuration launcherConf) { 955 injectCallback(context, launcherConf); 956 } 957 958 private void actionConfToLauncherConf(Configuration actionConf, Configuration launcherConf) { 959 for (String name : SPECIAL_PROPERTIES) { 960 if (actionConf.get(name) != null && launcherConf.get("oozie.launcher." + name) == null) { 961 launcherConf.set(name, actionConf.get(name)); 962 } 963 } 964 } 965 966 public void submitLauncher(FileSystem actionFs, final Context context, WorkflowAction action) throws ActionExecutorException { 967 YarnClient yarnClient = null; 968 try { 969 Path appPathRoot = new Path(context.getWorkflow().getAppPath()); 970 971 // app path could be a file 972 if (actionFs.isFile(appPathRoot)) { 973 appPathRoot = appPathRoot.getParent(); 974 } 975 976 Element actionXml = XmlUtils.parseXml(action.getConf()); 977 LOG.debug("ActionXML: {0}", action.getConf()); 978 979 // action job configuration 980 Configuration actionConf = loadHadoopDefaultResources(context, actionXml); 981 setupActionConf(actionConf, context, actionXml, appPathRoot); 982 addAppNameContext(action, context); 983 LOG.debug("Setting LibFilesArchives "); 984 setLibFilesArchives(context, actionXml, appPathRoot, actionConf); 985 986 String jobName = actionConf.get(HADOOP_JOB_NAME); 987 if (jobName == null || jobName.isEmpty()) { 988 jobName = XLog.format("oozie:action:T={0}:W={1}:A={2}:ID={3}", 989 getType(), context.getWorkflow().getAppName(), 990 action.getName(), context.getWorkflow().getId()); 991 actionConf.set(HADOOP_JOB_NAME, jobName); 992 } 993 994 injectActionCallback(context, actionConf); 995 996 if(actionConf.get(ACL_MODIFY_JOB) == null || actionConf.get(ACL_MODIFY_JOB).trim().equals("")) { 997 // ONLY in the case where user has not given the 998 // modify-job ACL specifically 999 if (context.getWorkflow().getAcl() != null) { 1000 // setting the group owning the Oozie job to allow anybody in that 1001 // group to modify the jobs. 1002 actionConf.set(ACL_MODIFY_JOB, context.getWorkflow().getAcl()); 1003 } 1004 } 1005 1006 Credentials credentials = new Credentials(); 1007 Configuration launcherConf = createLauncherConf(actionFs, context, action, actionXml, actionConf); 1008 yarnClient = createYarnClient(context, launcherConf); 1009 Map<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context, 1010 action, actionConf); 1011 if (UserGroupInformation.isSecurityEnabled()) { 1012 addHadoopCredentialPropertiesToActionConf(credentialsProperties); 1013 } 1014 // Adding if action need to set more credential tokens 1015 Configuration credentialsConf = new Configuration(false); 1016 XConfiguration.copy(actionConf, credentialsConf); 1017 setCredentialTokens(credentials, credentialsConf, context, action, credentialsProperties); 1018 1019 // copy back new entries from credentialsConf 1020 for (Entry<String, String> entry : credentialsConf) { 1021 if (actionConf.get(entry.getKey()) == null) { 1022 actionConf.set(entry.getKey(), entry.getValue()); 1023 } 1024 } 1025 String consoleUrl; 1026 String launcherId = LauncherHelper.getRecoveryId(launcherConf, context.getActionDir(), context 1027 .getRecoveryId()); 1028 1029 removeHBaseSettingFromOozieDefaultResource(launcherConf); 1030 removeHBaseSettingFromOozieDefaultResource(actionConf); 1031 1032 1033 boolean alreadyRunning = launcherId != null; 1034 1035 // if user-retry is on, always submit new launcher 1036 boolean isUserRetry = ((WorkflowActionBean)action).isUserRetry(); 1037 LOG.debug("Creating yarnClient for action {0}", action.getId()); 1038 1039 if (alreadyRunning && !isUserRetry) { 1040 try { 1041 ApplicationId appId = ConverterUtils.toApplicationId(launcherId); 1042 ApplicationReport report = yarnClient.getApplicationReport(appId); 1043 consoleUrl = report.getTrackingUrl(); 1044 } catch (RemoteException e) { 1045 // caught when the application id does not exist 1046 LOG.error("Got RemoteException from YARN", e); 1047 String jobTracker = launcherConf.get(HADOOP_YARN_RM); 1048 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA017", 1049 "unknown job [{0}@{1}], cannot recover", launcherId, jobTracker); 1050 } 1051 } 1052 else { 1053 YarnClientApplication newApp = yarnClient.createApplication(); 1054 ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); 1055 ApplicationSubmissionContext appContext = 1056 createAppSubmissionContext(appId, launcherConf, context, actionConf, action.getName(), 1057 credentials, actionXml); 1058 yarnClient.submitApplication(appContext); 1059 1060 launcherId = appId.toString(); 1061 LOG.debug("After submission get the launcherId [{0}]", launcherId); 1062 ApplicationReport appReport = yarnClient.getApplicationReport(appId); 1063 consoleUrl = appReport.getTrackingUrl(); 1064 } 1065 1066 String jobTracker = launcherConf.get(HADOOP_YARN_RM); 1067 context.setStartData(launcherId, jobTracker, consoleUrl); 1068 } 1069 catch (Exception ex) { 1070 throw convertException(ex); 1071 } 1072 finally { 1073 if (yarnClient != null) { 1074 Closeables.closeQuietly(yarnClient); 1075 } 1076 } 1077 } 1078 1079 private void removeHBaseSettingFromOozieDefaultResource(final Configuration jobConf) { 1080 final String[] propertySources = jobConf.getPropertySources(HbaseCredentials.HBASE_USE_DYNAMIC_JARS); 1081 if (propertySources != null && propertySources.length > 0 && 1082 propertySources[0].contains(HbaseCredentials.OOZIE_HBASE_CLIENT_SITE_XML)) { 1083 jobConf.unset(HbaseCredentials.HBASE_USE_DYNAMIC_JARS); 1084 LOG.debug(String.format("Unset [%s] inserted from default Oozie resource XML [%s]", 1085 HbaseCredentials.HBASE_USE_DYNAMIC_JARS, HbaseCredentials.OOZIE_HBASE_CLIENT_SITE_XML)); 1086 } 1087 } 1088 1089 protected void addAppNameContext(WorkflowAction action, Context context) { 1090 String oozieActionName = String.format("oozie:launcher:T=%s:W=%s:A=%s:ID=%s", 1091 getType(), 1092 context.getWorkflow().getAppName(), 1093 action.getName(), 1094 context.getWorkflow().getId()); 1095 context.setVar(OOZIE_ACTION_NAME, oozieActionName); 1096 } 1097 1098 protected String getAppName(Context context) { 1099 return context.getVar(OOZIE_ACTION_NAME); 1100 } 1101 1102 private void addHadoopCredentialPropertiesToActionConf(Map<String, CredentialsProperties> credentialsProperties) { 1103 LOG.info("Adding default credentials for action: hdfs, yarn and jhs"); 1104 addHadoopCredentialProperties(credentialsProperties, CredentialsProviderFactory.HDFS); 1105 addHadoopCredentialProperties(credentialsProperties, CredentialsProviderFactory.YARN); 1106 addHadoopCredentialProperties(credentialsProperties, CredentialsProviderFactory.JHS); 1107 } 1108 1109 private void addHadoopCredentialProperties(Map<String, CredentialsProperties> credentialsProperties, String type) { 1110 credentialsProperties.put(type, new CredentialsProperties(type, type)); 1111 } 1112 1113 private ApplicationSubmissionContext createAppSubmissionContext(final ApplicationId appId, 1114 final Configuration launcherJobConf, 1115 final Context actionContext, 1116 final Configuration actionConf, 1117 final String actionName, 1118 final Credentials credentials, 1119 final Element actionXml) 1120 throws IOException, HadoopAccessorException, URISyntaxException { 1121 1122 ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); 1123 1124 setResources(launcherJobConf, appContext); 1125 setPriority(launcherJobConf, appContext); 1126 setQueue(launcherJobConf, appContext); 1127 appContext.setApplicationId(appId); 1128 setApplicationName(actionContext, actionName, appContext); 1129 appContext.setApplicationType("Oozie Launcher"); 1130 setMaxAttempts(launcherJobConf, appContext); 1131 1132 ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); 1133 YarnACLHandler yarnACL = new YarnACLHandler(launcherJobConf); 1134 yarnACL.setACLs(amContainer); 1135 1136 final String user = actionContext.getWorkflow().getUser(); 1137 // Set the resources to localize 1138 Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); 1139 ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(launcherJobConf); 1140 MRApps.setupDistributedCache(launcherJobConf, localResources); 1141 // Add the Launcher and Action configs as Resources 1142 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class); 1143 launcherJobConf.set(LauncherAM.OOZIE_SUBMITTER_USER, user); 1144 LocalResource launcherJobConfLR = has.createLocalResourceForConfigurationFile(LauncherAM.LAUNCHER_JOB_CONF_XML, user, 1145 launcherJobConf, actionContext.getAppFileSystem().getUri(), actionContext.getActionDir()); 1146 localResources.put(LauncherAM.LAUNCHER_JOB_CONF_XML, launcherJobConfLR); 1147 LocalResource actionConfLR = has.createLocalResourceForConfigurationFile(LauncherAM.ACTION_CONF_XML, user, actionConf, 1148 actionContext.getAppFileSystem().getUri(), actionContext.getActionDir()); 1149 localResources.put(LauncherAM.ACTION_CONF_XML, actionConfLR); 1150 amContainer.setLocalResources(localResources); 1151 1152 setEnvironmentVariables(launcherJobConf, amContainer); 1153 1154 List<String> vargs = createCommand(launcherJobConf, actionContext); 1155 setJavaOpts(launcherJobConf, actionXml, vargs); 1156 vargs.add(LauncherAM.class.getCanonicalName()); 1157 vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT); 1158 vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDERR); 1159 StringBuilder mergedCommand = new StringBuilder(); 1160 for (CharSequence str : vargs) { 1161 mergedCommand.append(str).append(" "); 1162 } 1163 1164 List<String> vargsFinal = ImmutableList.of(mergedCommand.toString()); 1165 LOG.debug("Command to launch container for ApplicationMaster is: {0}", mergedCommand); 1166 amContainer.setCommands(vargsFinal); 1167 appContext.setAMContainerSpec(amContainer); 1168 1169 // Set tokens 1170 if (credentials != null) { 1171 DataOutputBuffer dob = new DataOutputBuffer(); 1172 credentials.writeTokenStorageToStream(dob); 1173 amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); 1174 } 1175 1176 appContext.setCancelTokensWhenComplete(true); 1177 1178 return appContext; 1179 } 1180 1181 private void setMaxAttempts(Configuration launcherJobConf, ApplicationSubmissionContext appContext) { 1182 int launcherMaxAttempts; 1183 final int defaultLauncherMaxAttempts = ConfigurationService.getInt(DEFAULT_LAUNCHER_MAX_ATTEMPS); 1184 if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_MAX_ATTEMPTS) != null) { 1185 try { 1186 launcherMaxAttempts = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_MAX_ATTEMPTS, 1187 defaultLauncherMaxAttempts); 1188 } catch (final NumberFormatException ignored) { 1189 launcherMaxAttempts = defaultLauncherMaxAttempts; 1190 } 1191 } else { 1192 LOG.warn("Invalid configuration value [{0}] defined for launcher max attempts count, using default [{1}].", 1193 launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_MAX_ATTEMPTS), 1194 defaultLauncherMaxAttempts); 1195 launcherMaxAttempts = defaultLauncherMaxAttempts; 1196 } 1197 1198 LOG.trace("Reading from configuration max attempts count of the Launcher AM. [launcherMaxAttempts={0}]", 1199 launcherMaxAttempts); 1200 1201 if (launcherMaxAttempts > 0) { 1202 LOG.trace("Setting max attempts of the Launcher AM. [launcherMaxAttempts={0}]", launcherMaxAttempts); 1203 appContext.setMaxAppAttempts(launcherMaxAttempts); 1204 } 1205 else { 1206 LOG.warn("Not setting max attempts of the Launcher AM, value is invalid. [launcherMaxAttempts={0}]", 1207 launcherMaxAttempts); 1208 } 1209 } 1210 1211 private List<String> createCommand(final Configuration launcherJobConf, final Context context) { 1212 final List<String> vargs = new ArrayList<String>(6); 1213 1214 String launcherLogLevel = launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_LOG_LEVEL_PROPERTY); 1215 if (Strings.isNullOrEmpty(launcherLogLevel)) { 1216 launcherLogLevel = "INFO"; 1217 } 1218 1219 vargs.add(Apps.crossPlatformify(ApplicationConstants.Environment.JAVA_HOME.toString()) 1220 + "/bin/java"); 1221 1222 vargs.add("-Dlog4j.configuration=container-log4j.properties"); 1223 vargs.add("-Dlog4j.debug=true"); 1224 vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); 1225 vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + 1024 * 1024); 1226 vargs.add("-Dhadoop.root.logger=" + launcherLogLevel + ",CLA"); 1227 vargs.add("-Dhadoop.root.logfile=" + TaskLog.LogName.SYSLOG); 1228 vargs.add("-Dsubmitter.user=" + context.getWorkflow().getUser()); 1229 1230 return vargs; 1231 } 1232 1233 private void setJavaOpts(Configuration launcherJobConf, Element actionXml, List<String> vargs) { 1234 // Note: for backward compatibility reasons, we have to support the <java-opts> tag inside the <java> action 1235 // If both java/java-opt(s) and launcher/java-opts are defined, we pick java/java-opts 1236 // We also display a warning to let users know that they should migrate their workflow 1237 StringBuilder javaOpts = new StringBuilder(); 1238 boolean oldJavaOpts = handleJavaOpts(actionXml, javaOpts); 1239 if (oldJavaOpts) { 1240 vargs.add(javaOpts.toString()); 1241 } 1242 1243 final String oozieLauncherJavaOpts = launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY); 1244 if (oozieLauncherJavaOpts != null) { 1245 if (oldJavaOpts) { 1246 LOG.warn("<java-opts> was defined inside the <launcher> tag -- ignored"); 1247 } else { 1248 vargs.add(oozieLauncherJavaOpts); 1249 } 1250 } 1251 } 1252 1253 private boolean handleJavaOpts(Element actionXml, StringBuilder javaOpts) { 1254 Namespace ns = actionXml.getNamespace(); 1255 boolean oldJavaOpts = false; 1256 @SuppressWarnings("unchecked") 1257 List<Element> javaopts = actionXml.getChildren("java-opt", ns); 1258 for (Element opt: javaopts) { 1259 javaOpts.append(" ").append(opt.getTextTrim()); 1260 oldJavaOpts = true; 1261 } 1262 Element opt = actionXml.getChild("java-opts", ns); 1263 if (opt != null) { 1264 javaOpts.append(" ").append(opt.getTextTrim()); 1265 oldJavaOpts = true; 1266 } 1267 1268 if (oldJavaOpts) { 1269 LOG.warn("Note: <java-opts> inside the action is used in the workflow. Please move <java-opts> tag under" 1270 + " the <launcher> element. See the documentation for details"); 1271 } 1272 return oldJavaOpts; 1273 } 1274 1275 private void setApplicationName(Context context, String actionName, ApplicationSubmissionContext appContext) { 1276 String jobName = XLog.format("oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(), 1277 context.getWorkflow().getAppName(), actionName, 1278 context.getWorkflow().getId()); 1279 appContext.setApplicationName(jobName); 1280 } 1281 1282 private void setEnvironmentVariables(Configuration launcherConf, ContainerLaunchContext amContainer) throws IOException { 1283 Map<String, String> env = new HashMap<>(); 1284 1285 final String oozieLauncherEnvProperty = launcherConf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY); 1286 if (oozieLauncherEnvProperty != null) { 1287 Map<String, String> environmentVars = extractEnvVarsFromOozieLauncherProps(oozieLauncherEnvProperty); 1288 env.putAll(environmentVars); 1289 } 1290 1291 // This adds the Hadoop jars to the classpath in the Launcher JVM 1292 ClasspathUtils.setupClasspath(env, launcherConf); 1293 1294 if (needToAddMapReduceToClassPath(launcherConf)) { 1295 ClasspathUtils.addMapReduceToClasspath(env, launcherConf); 1296 } 1297 1298 addActionSpecificEnvVars(env); 1299 amContainer.setEnvironment(ImmutableMap.copyOf(env)); 1300 } 1301 1302 private void setQueue(Configuration launcherJobConf, ApplicationSubmissionContext appContext) { 1303 String launcherQueueName; 1304 if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY) != null) { 1305 launcherQueueName = launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY); 1306 } else { 1307 launcherQueueName = Preconditions.checkNotNull( 1308 ConfigurationService.get(DEFAULT_LAUNCHER_QUEUE), "Default launcherQueueName is undefined"); 1309 } 1310 appContext.setQueue(launcherQueueName); 1311 } 1312 1313 private void setPriority(Configuration launcherJobConf, ApplicationSubmissionContext appContext) { 1314 int priority; 1315 if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_PRIORITY_PROPERTY) != null) { 1316 priority = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_PRIORITY_PROPERTY, -1); 1317 } else { 1318 int defaultPriority = ConfigurationService.getInt(DEFAULT_LAUNCHER_PRIORITY); 1319 priority = defaultPriority; 1320 } 1321 Priority pri = Records.newRecord(Priority.class); 1322 pri.setPriority(priority); 1323 appContext.setPriority(pri); 1324 } 1325 1326 private void setResources(Configuration launcherJobConf, ApplicationSubmissionContext appContext) { 1327 int memory; 1328 if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY) != null) { 1329 memory = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, -1); 1330 Preconditions.checkArgument(memory > 0, "Launcher memory is 0 or negative"); 1331 } else { 1332 int defaultMemory = ConfigurationService.getInt(DEFAULT_LAUNCHER_MEMORY_MB, -1); 1333 Preconditions.checkArgument(defaultMemory > 0, "Default launcher memory is 0 or negative"); 1334 memory = defaultMemory; 1335 } 1336 1337 int vcores; 1338 if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY) != null) { 1339 vcores = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, -1); 1340 Preconditions.checkArgument(vcores > 0, "Launcher vcores is 0 or negative"); 1341 } else { 1342 int defaultVcores = ConfigurationService.getInt(DEFAULT_LAUNCHER_VCORES); 1343 Preconditions.checkArgument(defaultVcores > 0, "Default launcher vcores is 0 or negative"); 1344 vcores = defaultVcores; 1345 } 1346 Resource resource = Resource.newInstance(memory, vcores); 1347 appContext.setResource(resource); 1348 } 1349 1350 private Map<String, String> extractEnvVarsFromOozieLauncherProps(String oozieLauncherEnvProperty) { 1351 Map<String, String> envMap = new LinkedHashMap<>(); 1352 for (String envVar : StringUtils.split(oozieLauncherEnvProperty, File.pathSeparatorChar)) { 1353 String[] env = StringUtils.split(envVar, '='); 1354 Preconditions.checkArgument(env.length == 2, "Invalid launcher setting for environment variables: \"%s\". " + 1355 "<env> should contain a list of ENV_VAR_NAME=VALUE separated by the '%s' character. " + 1356 "Example on Unix: A=foo1:B=foo2", oozieLauncherEnvProperty, File.pathSeparator); 1357 envMap.put(env[0], env[1]); 1358 } 1359 return envMap; 1360 } 1361 1362 Map<String, CredentialsProperties> setCredentialPropertyToActionConf(final Context context, 1363 final WorkflowAction action, 1364 final Configuration actionConf) throws Exception { 1365 final Map<String, CredentialsProperties> credPropertiesMap = new HashMap<>(); 1366 if (context == null || action == null) { 1367 LOG.warn("context or action is null"); 1368 return credPropertiesMap; 1369 } 1370 final XConfiguration wfJobConf = getWorkflowConf(context); 1371 final boolean skipCredentials = actionConf.getBoolean(OOZIE_CREDENTIALS_SKIP, 1372 wfJobConf.getBoolean(OOZIE_CREDENTIALS_SKIP, ConfigurationService.getBoolean(OOZIE_CREDENTIALS_SKIP))); 1373 if (skipCredentials) { 1374 LOG.info("Skipping credentials (" + OOZIE_CREDENTIALS_SKIP + "=true)"); 1375 } else { 1376 credPropertiesMap.putAll(getActionCredentialsProperties(context, action)); 1377 if (credPropertiesMap.isEmpty()) { 1378 LOG.warn("No credential properties found for action : " + action.getId() + ", cred : " + action.getCred()); 1379 return credPropertiesMap; 1380 } 1381 for (final Entry<String, CredentialsProperties> entry : credPropertiesMap.entrySet()) { 1382 if (entry.getValue() != null) { 1383 final CredentialsProperties prop = entry.getValue(); 1384 LOG.debug("Credential Properties set for action : " + action.getId()); 1385 for (final Entry<String, String> propEntry : prop.getProperties().entrySet()) { 1386 final String key = propEntry.getKey(); 1387 final String value = propEntry.getValue(); 1388 actionConf.set(key, value); 1389 LOG.debug("property : '" + key + "', value : '" + value + "'"); 1390 } 1391 } 1392 } 1393 } 1394 return credPropertiesMap; 1395 } 1396 1397 protected void setCredentialTokens(Credentials credentials, Configuration jobconf, Context context, WorkflowAction action, 1398 Map<String, CredentialsProperties> credPropertiesMap) throws Exception { 1399 if (!isValidCredentialTokensPreconditions(context, action, credPropertiesMap)) { 1400 LOG.debug("Not obtaining delegation token(s) as preconditions do not hold."); 1401 return; 1402 } 1403 1404 setActionTokenProperties(jobconf); 1405 // Make sure we're logged into Kerberos; if not, or near expiration, it will relogin 1406 CredentialsProviderFactory.ensureKerberosLogin(); 1407 for (Entry<String, CredentialsProperties> entry : credPropertiesMap.entrySet()) { 1408 String credName = entry.getKey(); 1409 CredentialsProperties credProps = entry.getValue(); 1410 if (credProps != null) { 1411 CredentialsProvider tokenProvider = CredentialsProviderFactory.getInstance() 1412 .createCredentialsProvider(credProps.getType()); 1413 if (tokenProvider != null) { 1414 tokenProvider.updateCredentials(credentials, jobconf, credProps, context); 1415 LOG.debug("Retrieved Credential '" + credName + "' for action " + action.getId()); 1416 } else { 1417 LOG.debug("Credentials object is null for name= " + credName + ", type=" + credProps.getType()); 1418 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA020", 1419 "Could not load credentials of type [{0}] with name [{1}]]; perhaps it was not defined" 1420 + " in oozie-site.xml?", credProps.getType(), credName); 1421 } 1422 } 1423 } 1424 } 1425 1426 private boolean isValidCredentialTokensPreconditions(final Context context, 1427 final WorkflowAction action, 1428 final Map<String, CredentialsProperties> credPropertiesMap) { 1429 return context != null && action != null && credPropertiesMap != null; 1430 } 1431 1432 /** 1433 * Subclasses may override this method in order to take additional actions required for obtaining credential token(s). 1434 * 1435 * @param jobconf workflow action configuration 1436 */ 1437 protected void setActionTokenProperties(final Configuration jobconf) { 1438 // nop 1439 } 1440 1441 protected HashMap<String, CredentialsProperties> getActionCredentialsProperties(Context context, 1442 WorkflowAction action) throws Exception { 1443 HashMap<String, CredentialsProperties> props = new HashMap<String, CredentialsProperties>(); 1444 if (context != null && action != null) { 1445 String credsInAction = action.getCred(); 1446 if (credsInAction != null) { 1447 LOG.debug("Get credential '" + credsInAction + "' properties for action : " + action.getId()); 1448 String[] credNames = credsInAction.split(","); 1449 for (String credName : credNames) { 1450 CredentialsProperties credProps = getCredProperties(context, credName); 1451 props.put(credName, credProps); 1452 } 1453 } 1454 } 1455 else { 1456 LOG.warn("context or action is null"); 1457 } 1458 return props; 1459 } 1460 1461 @SuppressWarnings("unchecked") 1462 protected CredentialsProperties getCredProperties(Context context, String credName) 1463 throws Exception { 1464 CredentialsProperties credProp = null; 1465 String workflowXml = ((WorkflowJobBean) context.getWorkflow()).getWorkflowInstance().getApp().getDefinition(); 1466 XConfiguration wfjobConf = getWorkflowConf(context); 1467 Element elementJob = XmlUtils.parseXml(workflowXml); 1468 Element credentials = elementJob.getChild("credentials", elementJob.getNamespace()); 1469 if (credentials != null) { 1470 for (Element credential : (List<Element>) credentials.getChildren("credential", credentials.getNamespace())) { 1471 String name = credential.getAttributeValue("name"); 1472 String type = credential.getAttributeValue("type"); 1473 LOG.debug("getCredProperties: Name: " + name + ", Type: " + type); 1474 if (name.equalsIgnoreCase(credName)) { 1475 credProp = new CredentialsProperties(name, type); 1476 for (Element property : (List<Element>) credential.getChildren("property", 1477 credential.getNamespace())) { 1478 String propertyName = property.getChildText("name", property.getNamespace()); 1479 String propertyValue = property.getChildText("value", property.getNamespace()); 1480 ELEvaluator eval = new ELEvaluator(); 1481 for (Map.Entry<String, String> entry : wfjobConf) { 1482 eval.setVariable(entry.getKey(), entry.getValue().trim()); 1483 } 1484 propertyName = eval.evaluate(propertyName, String.class); 1485 propertyValue = eval.evaluate(propertyValue, String.class); 1486 1487 credProp.getProperties().put(propertyName, propertyValue); 1488 LOG.debug("getCredProperties: Properties name :'" + propertyName + "', Value : '" 1489 + propertyValue + "'"); 1490 } 1491 } 1492 } 1493 if (credProp == null && credName != null) { 1494 throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "JA021", 1495 "Could not load credentials with name [{0}]].", credName); 1496 } 1497 } else { 1498 LOG.debug("credentials is null for the action"); 1499 } 1500 return credProp; 1501 } 1502 1503 @Override 1504 public void start(Context context, WorkflowAction action) throws ActionExecutorException { 1505 LogUtils.setLogInfo(action); 1506 try { 1507 LOG.info("Starting action. Getting Action File System"); 1508 FileSystem actionFs = context.getAppFileSystem(); 1509 LOG.debug("Preparing action Dir through copying " + context.getActionDir()); 1510 prepareActionDir(actionFs, context); 1511 LOG.debug("Action Dir is ready. Submitting the action "); 1512 submitLauncher(actionFs, context, action); 1513 LOG.debug("Action submit completed. Performing check "); 1514 check(context, action); 1515 LOG.debug("Action check is done after submission"); 1516 } 1517 catch (Exception ex) { 1518 throw convertException(ex); 1519 } 1520 } 1521 1522 @Override 1523 public void end(Context context, WorkflowAction action) throws ActionExecutorException { 1524 LOG.info("Action ended with external status [{0}]", action.getExternalStatus()); 1525 try { 1526 String externalStatus = action.getExternalStatus(); 1527 WorkflowAction.Status status = externalStatus.equals(SUCCEEDED) ? WorkflowAction.Status.OK 1528 : WorkflowAction.Status.ERROR; 1529 context.setEndData(status, getActionSignal(status)); 1530 } 1531 catch (Exception ex) { 1532 throw convertException(ex); 1533 } 1534 finally { 1535 try { 1536 FileSystem actionFs = context.getAppFileSystem(); 1537 cleanUpActionDir(actionFs, context); 1538 } 1539 catch (Exception ex) { 1540 throw convertException(ex); 1541 } 1542 } 1543 } 1544 1545 /** 1546 * Create job client object 1547 * 1548 * @param context 1549 * @param jobConf 1550 * @return JobClient 1551 * @throws HadoopAccessorException 1552 */ 1553 protected JobClient createJobClient(Context context, Configuration jobConf) throws HadoopAccessorException { 1554 String user = context.getWorkflow().getUser(); 1555 return Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf); 1556 } 1557 1558 /** 1559 * Create yarn client object 1560 * 1561 * @param context 1562 * @param jobConf 1563 * @return YarnClient 1564 * @throws HadoopAccessorException 1565 */ 1566 protected YarnClient createYarnClient(Context context, Configuration jobConf) throws HadoopAccessorException { 1567 String user = context.getWorkflow().getUser(); 1568 return Services.get().get(HadoopAccessorService.class).createYarnClient(user, jobConf); 1569 } 1570 1571 /** 1572 * Useful for overriding in actions that do subsequent job runs 1573 * such as the MapReduce Action, where the launcher job is not the 1574 * actual job that then gets monitored. 1575 * @param action 1576 * @return external ID. 1577 */ 1578 protected String getActualExternalId(WorkflowAction action) { 1579 return action.getExternalId(); 1580 } 1581 1582 /** 1583 * If returns true, it means that we have to add Hadoop MR jars to the classpath. 1584 * Subclasses should override this method if necessary. By default we don't add 1585 * MR jars to the classpath. 1586 * 1587 * <p>Configuration properties are read either from {@code launcherConf}, or, if not present, falling back to 1588 * {@link ConfigurationService#getBoolean(Configuration, String)}. 1589 * 1590 * @return false by default 1591 * @param launcherConf the launcher {@link Configuration} that is used on first lookup 1592 */ 1593 private boolean needToAddMapReduceToClassPath(final Configuration launcherConf) { 1594 LOG.debug("Calculating whether to add MapReduce JARs to classpath. [type={0};this={1}]", getType(), this); 1595 1596 final boolean defaultValue = ConfigurationService.getBooleanOrDefault( 1597 launcherConf, 1598 OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, 1599 false); 1600 LOG.debug("Default value for [{0}] is [{1}]", OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY, defaultValue); 1601 1602 final String configurationKey = OOZIE_LAUNCHER_ADD_MAPREDUCE_TO_CLASSPATH_PROPERTY + "." + getType(); 1603 final boolean configuredValue = ConfigurationService.getBooleanOrDefault(launcherConf, configurationKey, defaultValue); 1604 LOG.debug("Configured value for [{0}] is [{1}]", configurationKey, configuredValue); 1605 1606 return configuredValue; 1607 } 1608 1609 /** 1610 * Adds action-specific environment variables. Default implementation is no-op. 1611 * Subclasses should override this method if necessary. 1612 * @param env action specific environment variables stored as Map of key and value. 1613 */ 1614 protected void addActionSpecificEnvVars(Map<String, String> env) { 1615 // nop 1616 } 1617 1618 @Override 1619 public void check(Context context, WorkflowAction action) throws ActionExecutorException { 1620 boolean fallback = false; 1621 LOG = XLog.resetPrefix(LOG); 1622 LogUtils.setLogInfo(action); 1623 YarnClient yarnClient = null; 1624 try { 1625 Element actionXml = XmlUtils.parseXml(action.getConf()); 1626 Configuration jobConf = createBaseHadoopConf(context, actionXml); 1627 FileSystem actionFs = context.getAppFileSystem(); 1628 yarnClient = createYarnClient(context, jobConf); 1629 FinalApplicationStatus appStatus = null; 1630 try { 1631 ApplicationReport appReport = 1632 yarnClient.getApplicationReport(ConverterUtils.toApplicationId(action.getExternalId())); 1633 YarnApplicationState appState = appReport.getYarnApplicationState(); 1634 if (appState == YarnApplicationState.FAILED || appState == YarnApplicationState.FINISHED 1635 || appState == YarnApplicationState.KILLED) { 1636 appStatus = appReport.getFinalApplicationStatus(); 1637 } 1638 1639 } catch (Exception ye) { 1640 LOG.warn("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye); 1641 // Fallback to action data file if we can't find the Launcher AM (maybe it got purged) 1642 fallback = true; 1643 } 1644 if (appStatus != null || fallback) { 1645 Path actionDir = context.getActionDir(); 1646 // load sequence file into object 1647 Map<String, String> actionData = LauncherHelper.getActionData(actionFs, actionDir, jobConf); 1648 if (fallback) { 1649 String finalStatus = actionData.get(LauncherAM.ACTION_DATA_FINAL_STATUS); 1650 if (finalStatus != null) { 1651 appStatus = FinalApplicationStatus.valueOf(finalStatus); 1652 } else { 1653 context.setExecutionData(FAILED, null); 1654 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "JA017", 1655 "Unknown hadoop job [{0}] associated with action [{1}] and couldn't determine status from" + 1656 " action data. Failing this action!", action.getExternalId(), action.getId()); 1657 } 1658 } 1659 1660 String externalID = actionData.get(LauncherAM.ACTION_DATA_NEW_ID); // MapReduce was launched 1661 if (externalID != null) { 1662 context.setExternalChildIDs(externalID); 1663 LOG.info(XLog.STD, "Hadoop Job was launched : [{0}]", externalID); 1664 } 1665 1666 // Multiple child IDs - Pig or Hive action 1667 String externalIDs = actionData.get(LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS); 1668 if (externalIDs != null) { 1669 context.setExternalChildIDs(externalIDs); 1670 LOG.info(XLog.STD, "External Child IDs : [{0}]", externalIDs); 1671 1672 } 1673 1674 LOG.info(XLog.STD, "action completed, external ID [{0}]", action.getExternalId()); 1675 context.setExecutionData(appStatus.toString(), null); 1676 if (appStatus == FinalApplicationStatus.SUCCEEDED) { 1677 if (getCaptureOutput(action) && LauncherHelper.hasOutputData(actionData)) { 1678 context.setExecutionData(SUCCEEDED, PropertiesUtils.stringToProperties(actionData 1679 .get(LauncherAM.ACTION_DATA_OUTPUT_PROPS))); 1680 LOG.info(XLog.STD, "action produced output"); 1681 } 1682 else { 1683 context.setExecutionData(SUCCEEDED, null); 1684 } 1685 if (LauncherHelper.hasStatsData(actionData)) { 1686 context.setExecutionStats(actionData.get(LauncherAM.ACTION_DATA_STATS)); 1687 LOG.info(XLog.STD, "action produced stats"); 1688 } 1689 getActionData(actionFs, action, context); 1690 } 1691 else { 1692 String errorReason; 1693 if (actionData.containsKey(LauncherAM.ACTION_DATA_ERROR_PROPS)) { 1694 Properties props = PropertiesUtils.stringToProperties(actionData 1695 .get(LauncherAM.ACTION_DATA_ERROR_PROPS)); 1696 String errorCode = props.getProperty("error.code"); 1697 if ("0".equals(errorCode)) { 1698 errorCode = "JA018"; 1699 } 1700 if ("-1".equals(errorCode)) { 1701 errorCode = "JA019"; 1702 } 1703 errorReason = props.getProperty("error.reason"); 1704 LOG.warn("Launcher ERROR, reason: {0}", errorReason); 1705 String exMsg = props.getProperty("exception.message"); 1706 String errorInfo = (exMsg != null) ? exMsg : errorReason; 1707 context.setErrorInfo(errorCode, errorInfo); 1708 String exStackTrace = props.getProperty("exception.stacktrace"); 1709 if (exMsg != null) { 1710 LOG.warn("Launcher exception: {0}{E}{1}", exMsg, exStackTrace); 1711 } 1712 } 1713 else { 1714 errorReason = XLog.format("Launcher AM died, check Hadoop LOG for job [{0}:{1}]", action 1715 .getTrackerUri(), action.getExternalId()); 1716 LOG.warn(errorReason); 1717 } 1718 context.setExecutionData(FAILED_KILLED, null); 1719 } 1720 } 1721 else { 1722 context.setExternalStatus(YarnApplicationState.RUNNING.toString()); 1723 LOG.info(XLog.STD, "checking action, hadoop job ID [{0}] status [RUNNING]", 1724 action.getExternalId()); 1725 } 1726 } 1727 catch (Exception ex) { 1728 LOG.warn("Exception in check(). Message[{0}]", ex.getMessage(), ex); 1729 throw convertException(ex); 1730 } 1731 finally { 1732 if (yarnClient != null) { 1733 IOUtils.closeQuietly(yarnClient); 1734 } 1735 } 1736 } 1737 1738 /** 1739 * Get the output data of an action. Subclasses should override this method 1740 * to get action specific output data. 1741 * @param actionFs the FileSystem object 1742 * @param action the Workflow action 1743 * @param context executor context 1744 * @throws org.apache.oozie.service.HadoopAccessorException 1745 * @throws org.jdom.JDOMException 1746 * @throws java.io.IOException 1747 * @throws java.net.URISyntaxException 1748 * 1749 */ 1750 protected void getActionData(FileSystem actionFs, WorkflowAction action, Context context) 1751 throws HadoopAccessorException, JDOMException, IOException, URISyntaxException { 1752 } 1753 1754 protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException { 1755 Element eConf = XmlUtils.parseXml(action.getConf()); 1756 Namespace ns = eConf.getNamespace(); 1757 Element captureOutput = eConf.getChild("capture-output", ns); 1758 return captureOutput != null; 1759 } 1760 1761 @Override 1762 public void kill(Context context, WorkflowAction action) throws ActionExecutorException { 1763 YarnClient yarnClient = null; 1764 try { 1765 Element actionXml = XmlUtils.parseXml(action.getConf()); 1766 final Configuration jobConf = createBaseHadoopConf(context, actionXml); 1767 String launcherTag = LauncherHelper.getActionYarnTag(jobConf, context.getWorkflow().getParentId(), action); 1768 jobConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, LauncherHelper.getTag(launcherTag)); 1769 yarnClient = createYarnClient(context, jobConf); 1770 if(action.getExternalId() != null) { 1771 try { 1772 LOG.info("Killing action {0}'s external application {1}", action.getId(), action.getExternalId()); 1773 yarnClient.killApplication(ConverterUtils.toApplicationId(action.getExternalId())); 1774 } catch (Exception e) { 1775 LOG.warn("Could not kill {0}", action.getExternalId(), e); 1776 } 1777 } 1778 String externalChildIDs = action.getExternalChildIDs(); 1779 if(externalChildIDs != null) { 1780 for(String childId : externalChildIDs.split(",")) { 1781 try { 1782 LOG.info("Killing action {0}'s external child application {1}", action.getId(), childId); 1783 yarnClient.killApplication(ConverterUtils.toApplicationId(childId.trim())); 1784 } catch (Exception e) { 1785 LOG.warn("Could not kill external child of {0}, {1}", action.getExternalId(), 1786 childId, e); 1787 } 1788 } 1789 } 1790 for(ApplicationId id : LauncherMain.getChildYarnJobs(jobConf, ApplicationsRequestScope.ALL, 1791 action.getStartTime().getTime())){ 1792 try { 1793 LOG.info("Killing action {0}'s external child application {1} based on tags", 1794 action.getId(), id.toString()); 1795 yarnClient.killApplication(id); 1796 } catch (Exception e) { 1797 LOG.warn("Could not kill child of {0}, {1}", action.getExternalId(), id, e); 1798 } 1799 } 1800 1801 context.setExternalStatus(KILLED); 1802 context.setExecutionData(KILLED, null); 1803 } catch (Exception ex) { 1804 LOG.error("Error when killing YARN application", ex); 1805 throw convertException(ex); 1806 } finally { 1807 try { 1808 FileSystem actionFs = context.getAppFileSystem(); 1809 cleanUpActionDir(actionFs, context); 1810 Closeables.closeQuietly(yarnClient); 1811 } catch (Exception ex) { 1812 LOG.error("Error when cleaning up action dir", ex); 1813 throw convertException(ex); 1814 } 1815 } 1816 } 1817 1818 private static Set<String> FINAL_STATUS = new HashSet<String>(); 1819 1820 static { 1821 FINAL_STATUS.add(SUCCEEDED); 1822 FINAL_STATUS.add(KILLED); 1823 FINAL_STATUS.add(FAILED); 1824 FINAL_STATUS.add(FAILED_KILLED); 1825 } 1826 1827 @Override 1828 public boolean isCompleted(String externalStatus) { 1829 return FINAL_STATUS.contains(externalStatus); 1830 } 1831 1832 1833 /** 1834 * Return the sharelib names for the action. 1835 * See {@link SharelibResolver} for details. 1836 * 1837 * @param context executor context. 1838 * @param actionXml the action xml. 1839 * @param conf action configuration. 1840 * @return the action sharelib names. 1841 */ 1842 protected String[] getShareLibNames(final Context context, final Element actionXml, Configuration conf) { 1843 final String sharelibFromConfigurationProperty = ACTION_SHARELIB_FOR + getType(); 1844 try { 1845 final String defaultShareLibName = getDefaultShareLibName(actionXml); 1846 final Configuration oozieServerConfiguration = Services.get().get(ConfigurationService.class).getConf(); 1847 final XConfiguration workflowConfiguration = getWorkflowConf(context); 1848 return new SharelibResolver(sharelibFromConfigurationProperty, conf, workflowConfiguration, 1849 oozieServerConfiguration, defaultShareLibName).resolve(); 1850 } catch (IOException ex) { 1851 throw new RuntimeException("Can't get workflow configuration: " + ex.toString(), ex); 1852 } 1853 } 1854 1855 1856 /** 1857 * Returns the default sharelib name for the action if any. 1858 * 1859 * @param actionXml the action XML fragment. 1860 * @return the sharelib name for the action, <code>NULL</code> if none. 1861 */ 1862 protected String getDefaultShareLibName(Element actionXml) { 1863 return null; 1864 } 1865 1866 public String[] getShareLibFilesForActionConf() { 1867 return null; 1868 } 1869 1870 /** 1871 * Sets some data for the action on completion 1872 * 1873 * @param context executor context 1874 * @param actionFs the FileSystem object 1875 * @throws java.io.IOException 1876 * @throws org.apache.oozie.service.HadoopAccessorException 1877 * @throws java.net.URISyntaxException 1878 */ 1879 protected void setActionCompletionData(Context context, FileSystem actionFs) throws IOException, 1880 HadoopAccessorException, URISyntaxException { 1881 } 1882 1883 private void injectJobInfo(Configuration launcherJobConf, Configuration actionConf, Context context, WorkflowAction action) { 1884 if (OozieJobInfo.isJobInfoEnabled()) { 1885 try { 1886 OozieJobInfo jobInfo = new OozieJobInfo(actionConf, context, action); 1887 String jobInfoStr = jobInfo.getJobInfo(); 1888 launcherJobConf.set(OozieJobInfo.JOB_INFO_KEY, jobInfoStr + "launcher=true"); 1889 actionConf.set(OozieJobInfo.JOB_INFO_KEY, jobInfoStr + "launcher=false"); 1890 } 1891 catch (Exception e) { 1892 // Just job info, should not impact the execution. 1893 LOG.error("Error while populating job info", e); 1894 } 1895 } 1896 } 1897 1898 @Override 1899 public boolean requiresNameNodeJobTracker() { 1900 return true; 1901 } 1902 1903 @Override 1904 public boolean supportsConfigurationJobXML() { 1905 return true; 1906 } 1907 1908 private XConfiguration getWorkflowConf(Context context) throws IOException { 1909 if (workflowConf == null) { 1910 workflowConf = new XConfiguration(new StringReader(context.getWorkflow().getConf())); 1911 } 1912 return workflowConf; 1913 1914 } 1915 1916 private String getActionTypeLauncherPrefix() { 1917 return "oozie.action." + getType() + ".launcher."; 1918 } 1919}