001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.command.coord; 020 021import java.io.IOException; 022import java.io.StringReader; 023import java.net.URI; 024import java.net.URISyntaxException; 025import java.text.ParseException; 026import java.util.ArrayList; 027import java.util.TimeZone; 028import java.util.List; 029import java.util.Date; 030import java.util.Calendar; 031 032import org.apache.hadoop.conf.Configuration; 033import org.apache.oozie.CoordinatorActionBean; 034import org.apache.oozie.ErrorCode; 035import org.apache.oozie.client.CoordinatorAction; 036import org.apache.oozie.client.OozieClient; 037import org.apache.oozie.command.CommandException; 038import org.apache.oozie.coord.CoordELEvaluator; 039import org.apache.oozie.coord.CoordELFunctions; 040import org.apache.oozie.coord.CoordUtils; 041import org.apache.oozie.coord.CoordinatorJobException; 042import org.apache.oozie.coord.SyncCoordAction; 043import org.apache.oozie.coord.TimeUnit; 044import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluatorUtil; 045import org.apache.oozie.coord.input.dependency.CoordInputDependency; 046import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluator; 047import org.apache.oozie.coord.input.dependency.CoordInputDependencyFactory; 048import org.apache.oozie.coord.input.dependency.CoordInputInstance; 049import org.apache.oozie.dependency.ActionDependency; 050import org.apache.oozie.dependency.DependencyChecker; 051import org.apache.oozie.dependency.URIHandler; 052import org.apache.oozie.dependency.URIHandler.DependencyType; 053import org.apache.oozie.dependency.URIHandlerException; 054import org.apache.oozie.service.Services; 055import org.apache.oozie.service.URIHandlerService; 056import org.apache.oozie.service.UUIDService; 057import org.apache.oozie.util.DateUtils; 058import org.apache.oozie.util.ELEvaluator; 059import org.apache.oozie.util.Pair; 060import org.apache.oozie.util.ParamChecker; 061import org.apache.oozie.util.XConfiguration; 062import org.apache.oozie.util.XmlUtils; 063import org.jdom.Attribute; 064import org.jdom.Element; 065import org.jdom.JDOMException; 066import org.quartz.CronExpression; 067import org.apache.commons.lang.StringUtils; 068import org.apache.oozie.CoordinatorJobBean; 069 070public class CoordCommandUtils { 071 public static final int CURRENT = 0; 072 public static final int LATEST = 1; 073 public static final int FUTURE = 2; 074 public static final int OFFSET = 3; 075 public static final int ABSOLUTE = 4; 076 public static final int ENDOFMONTHS = 5; 077 public static final int ENDOFWEEKS = 6; 078 public static final int ENDOFDAYS = 7; 079 public static final int UNEXPECTED = -1; 080 081 public static final String RESOLVED_UNRESOLVED_SEPARATOR = "!!"; 082 public static final String UNRESOLVED_INSTANCES_TAG = "unresolved-instances"; 083 084 /** 085 * parse a function like coord:latest(n)/future() and return the 'n'. 086 * <p> 087 * 088 * @param function 089 * @param restArg 090 * @return int instanceNumber 091 * @throws Exception 092 */ 093 public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception { 094 int funcType = getFuncType(function); 095 if (funcType == ABSOLUTE) { 096 return funcType; 097 } 098 if (funcType == CURRENT || funcType == LATEST || funcType == ENDOFMONTHS || funcType == ENDOFWEEKS 099 || funcType == ENDOFDAYS) { 100 return parseOneArg(function); 101 } 102 else { 103 return parseMoreArgs(function, restArg); 104 } 105 } 106 107 /** 108 * Evaluates function for coord-action-create-inst tag 109 * @param event 110 * @param appInst 111 * @param conf 112 * @param function 113 * @return evaluation result 114 * @throws Exception 115 */ 116 private static String evaluateInstanceFunction(Element event, SyncCoordAction appInst, Configuration conf, 117 String function) throws Exception { 118 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf); 119 return CoordELFunctions.evalAndWrap(eval, function); 120 } 121 122 public static int parseOneArg(String funcName) throws Exception { 123 int firstPos = funcName.indexOf("("); 124 int lastPos = funcName.lastIndexOf(")"); 125 if (firstPos >= 0 && lastPos > firstPos) { 126 String tmp = funcName.substring(firstPos + 1, lastPos).trim(); 127 if (tmp.length() > 0) { 128 return (int) Double.parseDouble(tmp); 129 } 130 } 131 throw new RuntimeException("Unformatted function :" + funcName); 132 } 133 134 public static String parseOneStringArg(String funcName) throws Exception { 135 int firstPos = funcName.indexOf("("); 136 int lastPos = funcName.lastIndexOf(")"); 137 if (firstPos >= 0 && lastPos > firstPos) { 138 return funcName.substring(firstPos + 1, lastPos).trim(); 139 } 140 throw new RuntimeException("Unformatted function :" + funcName); 141 } 142 143 private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception { 144 int firstPos = funcName.indexOf("("); 145 int secondPos = funcName.lastIndexOf(","); 146 int lastPos = funcName.lastIndexOf(")"); 147 if (firstPos >= 0 && secondPos > firstPos) { 148 String tmp = funcName.substring(firstPos + 1, secondPos).trim(); 149 if (tmp.length() > 0) { 150 restArg.append(funcName.substring(secondPos + 1, lastPos).trim()); 151 return (int) Double.parseDouble(tmp); 152 } 153 } 154 throw new RuntimeException("Unformatted function :" + funcName); 155 } 156 157 /** 158 * @param function EL function name 159 * @return type of EL function 160 */ 161 public static int getFuncType(String function) { 162 if (function.indexOf("current") >= 0) { 163 return CURRENT; 164 } 165 else if (function.indexOf("latest") >= 0) { 166 return LATEST; 167 } 168 else if (function.indexOf("future") >= 0) { 169 return FUTURE; 170 } 171 else if (function.indexOf("offset") >= 0) { 172 return OFFSET; 173 } 174 else if (function.indexOf("absolute") >= 0) { 175 return ABSOLUTE; 176 } 177 else if (function.indexOf("endOfMonths") >= 0) { 178 return ENDOFMONTHS; 179 } 180 else if (function.indexOf("endOfWeeks") >= 0) { 181 return ENDOFWEEKS; 182 } 183 else if (function.indexOf("endOfDays") >= 0) { 184 return ENDOFDAYS; 185 } 186 return UNEXPECTED; 187 // throw new RuntimeException("Unexpected instance name "+ function); 188 } 189 190 /** 191 * @param startInst EL function name 192 * @param endInst EL function name 193 * @throws CommandException if both are not the same function 194 */ 195 public static void checkIfBothSameType(String startInst, String endInst) throws CommandException { 196 if (getFuncType(startInst) != getFuncType(endInst)) { 197 if (getFuncType(startInst) == ABSOLUTE || getFuncType(startInst) == ENDOFMONTHS 198 || getFuncType(startInst) == ENDOFWEEKS || getFuncType(startInst) == ENDOFDAYS) { 199 if (getFuncType(endInst) != CURRENT) { 200 throw new CommandException(ErrorCode.E1010, 201 "Only start-instance as absolute/endOfMonths/endOfWeeks/endOfDays and end-instance as current is" 202 + " supported." 203 + " start = " + startInst + " end = " + endInst); 204 } 205 } 206 else { 207 throw new CommandException(ErrorCode.E1010, 208 " start-instance and end-instance both should be either latest or current or future or offset\n" 209 + " start " + startInst + " and end " + endInst); 210 } 211 } 212 } 213 214 215 /** 216 * Resolve list of <instance> </instance> tags. 217 * 218 * @param event 219 * @param instances 220 * @param actionInst 221 * @param conf 222 * @param eval ELEvalautor 223 * @throws Exception 224 */ 225 public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst, 226 Configuration conf, ELEvaluator eval) throws Exception { 227 for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) { 228 229 if (instances.length() > 0) { 230 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 231 } 232 instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval)); 233 } 234 event.removeChildren("instance", event.getNamespace()); 235 } 236 237 /** 238 * Resolve <start-instance> <end-insatnce> tag. Don't resolve any 239 * latest()/future() 240 * 241 * @param event 242 * @param instances 243 * @param appInst 244 * @param conf 245 * @param eval ELEvalautor 246 * @throws Exception 247 */ 248 public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst, 249 Configuration conf, ELEvaluator eval) throws Exception { 250 Element eStartInst = event.getChild("start-instance", event.getNamespace()); 251 Element eEndInst = event.getChild("end-instance", event.getNamespace()); 252 if (eStartInst != null && eEndInst != null) { 253 String strStart = evaluateInstanceFunction(event, appInst, conf, eStartInst.getTextTrim()); 254 String strEnd = evaluateInstanceFunction(event, appInst, conf, eEndInst.getTextTrim()); 255 checkIfBothSameType(strStart, strEnd); 256 StringBuilder restArg = new StringBuilder(); // To store rest 257 // arguments for 258 // future 259 // function 260 261 int startIndex = getInstanceNumber(strStart, restArg); 262 String startRestArg = restArg.toString(); 263 restArg.delete(0, restArg.length()); 264 int endIndex = getInstanceNumber(strEnd, restArg); 265 String endRestArg = restArg.toString(); 266 int funcType = getFuncType(strStart); 267 268 if (funcType == ABSOLUTE) { 269 resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, 270 parseOneStringArg(strStart)); 271 } 272 else if (funcType == ENDOFMONTHS) { 273 resolveInstanceRangeEndOfDuration(TimeUnit.MONTH, event, instances, appInst, conf, eval, strStart, 274 startIndex, endIndex); 275 } 276 else if (funcType == ENDOFWEEKS) { 277 resolveInstanceRangeEndOfDuration(TimeUnit.WEEK, event, instances, appInst, conf, eval, strStart, 278 startIndex, endIndex); 279 } 280 else if (funcType == ENDOFDAYS) { 281 resolveInstanceRangeEndOfDuration(TimeUnit.DAY, event, instances, appInst, conf, eval, strStart, 282 startIndex, endIndex); 283 } 284 else { 285 if (funcType == OFFSET) { 286 TimeUnit startU = TimeUnit.valueOf(startRestArg); 287 TimeUnit endU = TimeUnit.valueOf(endRestArg); 288 if (startU.getCalendarUnit() * startIndex > endU.getCalendarUnit() * endIndex) { 289 throw new CommandException(ErrorCode.E1010, 290 " start-instance should be equal or earlier than the end-instance \n" 291 + XmlUtils.prettyPrint(event)); 292 } 293 Calendar startCal = CoordELFunctions.resolveOffsetRawTime(startIndex, startU, eval); 294 Calendar endCal = CoordELFunctions.resolveOffsetRawTime(endIndex, endU, eval); 295 if (startCal != null && endCal != null) { 296 List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval); 297 for (int i = expandedFreqs.size() - 1; i >= 0; i--) { 298 //we need to use DS timeout, bcz expandOffsetTimes will expand offset in Freqs in DS timeunit 299 String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i) 300 + ", \"" + CoordELFunctions.getDSTimeUnit(eval) + "\")}", appInst, conf, eval); 301 if (matInstance == null || matInstance.length() == 0) { 302 // Earlier than dataset's initial instance 303 break; 304 } 305 if (instances.length() > 0) { 306 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 307 } 308 instances.append(matInstance); 309 } 310 } 311 } 312 else { 313 if (startIndex > endIndex) { 314 throw new CommandException(ErrorCode.E1010, 315 " start-instance should be equal or earlier than the end-instance \n" 316 + XmlUtils.prettyPrint(event)); 317 } 318 if (funcType == CURRENT) { 319 // Everything could be resolved NOW. no latest() ELs 320 String matInstance = materializeInstance(event, "${coord:currentRange(" + startIndex + "," 321 + endIndex + ")}", appInst, conf, eval); 322 if (matInstance != null && !matInstance.isEmpty()) { 323 if (instances.length() > 0) { 324 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 325 } 326 instances.append(matInstance); 327 } 328 } 329 330 else { // latest(n)/future() EL is present 331 if (funcType == LATEST) { 332 instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex) 333 .append(")}"); 334 } 335 else if (funcType == FUTURE) { 336 instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex) 337 .append(",'").append(endRestArg).append("')}"); 338 } 339 } 340 } 341 } 342 // Remove start-instance and end-instances 343 event.removeChild("start-instance", event.getNamespace()); 344 event.removeChild("end-instance", event.getNamespace()); 345 } 346 } 347 348 private static void resolveAbsoluteRange(Element event, StringBuilder instances, SyncCoordAction appInst, 349 Configuration conf, ELEvaluator eval, String strStart, int endIndex, String rangeStr) throws Exception { 350 StringBuffer bf = new StringBuffer(); 351 bf.append("${coord:absoluteRange(\"").append(rangeStr).append("\",") 352 .append(endIndex).append(")}"); 353 String matInstance = materializeInstance(event, bf.toString(), appInst, conf, eval); 354 if (matInstance != null && !matInstance.isEmpty()) { 355 if (instances.length() > 0) { 356 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 357 } 358 instances.append(matInstance); 359 } 360 } 361 362 private static void resolveInstanceRangeEndOfDuration(TimeUnit duration, Element event, StringBuilder instances, 363 SyncCoordAction appInst, Configuration conf, ELEvaluator eval, String strStart, int startIndex, 364 int endIndex) throws Exception { 365 Calendar startInstance = new StartInstanceFinder(startIndex, duration, CoordELFunctions.getDatasetTZ(eval), 366 appInst.getNominalTime()).getStartInstance(); 367 resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, 368 DateUtils.formatDateOozieTZ(startInstance)); 369 } 370 371 /** 372 * Materialize one instance like current(-2) 373 * 374 * @param event : <data-in> 375 * @param expr : instance like current(-1) 376 * @param appInst : application specific info 377 * @param conf 378 * @param evalInst :ELEvaluator 379 * @return materialized date string 380 * @throws Exception 381 */ 382 public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf, 383 ELEvaluator evalInst) throws Exception { 384 if (event == null) { 385 return null; 386 } 387 // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, 388 // appInst, conf); 389 return CoordELFunctions.evalAndWrap(evalInst, expr); 390 } 391 392 /** 393 * Create two new tags with <uris> and <unresolved-instances>. 394 * 395 * @param event 396 * @param instances 397 * @throws Exception 398 */ 399 private static String separateResolvedAndUnresolved(Element event, StringBuilder instances) 400 throws Exception { 401 StringBuilder unresolvedInstances = new StringBuilder(); 402 StringBuilder urisWithDoneFlag = new StringBuilder(); 403 StringBuilder depList = new StringBuilder(); 404 String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag); 405 if (uris.length() > 0) { 406 Element uriInstance = new Element("uris", event.getNamespace()); 407 uriInstance.addContent(uris); 408 event.getContent().add(1, uriInstance); 409 if (depList.length() > 0) { 410 depList.append(CoordELFunctions.INSTANCE_SEPARATOR); 411 } 412 depList.append(urisWithDoneFlag); 413 } 414 if (unresolvedInstances.length() > 0) { 415 Element elemInstance = new Element(UNRESOLVED_INSTANCES_TAG, event.getNamespace()); 416 elemInstance.addContent(unresolvedInstances.toString()); 417 event.getContent().add(1, elemInstance); 418 } 419 return depList.toString(); 420 } 421 422 /** 423 * The function create a list of URIs separated by "," using the instances 424 * time stamp and URI-template 425 * 426 * @param event : <data-in> event 427 * @param instances : List of time stamp separated by "," 428 * @param unresolvedInstances : list of instance with latest function 429 * @param urisWithDoneFlag : list of URIs with the done flag appended 430 * @return : list of URIs separated by ";" as a string. 431 * @throws Exception 432 */ 433 public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances, 434 StringBuilder urisWithDoneFlag) throws Exception { 435 if (instances == null || instances.length() == 0) { 436 return ""; 437 } 438 String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR); 439 StringBuilder uris = new StringBuilder(); 440 441 Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag", 442 event.getNamespace()); 443 URIHandlerService uriService = Services.get().get(URIHandlerService.class); 444 445 for (int i = 0; i < instanceList.length; i++) { 446 if (instanceList[i].trim().length() == 0) { 447 continue; 448 } 449 int funcType = getFuncType(instanceList[i]); 450 if (funcType == LATEST || funcType == FUTURE) { 451 if (unresolvedInstances.length() > 0) { 452 unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR); 453 } 454 unresolvedInstances.append(instanceList[i]); 455 continue; 456 } 457 ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]); 458 if (uris.length() > 0) { 459 uris.append(CoordELFunctions.INSTANCE_SEPARATOR); 460 urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR); 461 } 462 463 String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace()) 464 .getChild("uri-template", event.getNamespace()).getTextTrim()); 465 URIHandler uriHandler = uriService.getURIHandler(uriPath); 466 uriHandler.validate(uriPath); 467 uris.append(uriPath); 468 urisWithDoneFlag.append(uriHandler.getURIWithDoneFlag(uriPath, CoordUtils.getDoneFlag(doneFlagElement))); 469 } 470 return uris.toString(); 471 } 472 473 /** 474 * @param eAction 475 * @param coordAction 476 * @param conf 477 * @return boolean to determine whether the SLA element is present or not 478 * @throws CoordinatorJobException 479 */ 480 public static boolean materializeSLA(Element eAction, CoordinatorActionBean coordAction, Configuration conf) 481 throws CoordinatorJobException { 482 Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla")); 483 if (eSla == null) { 484 // eAppXml.getNamespace("sla")); 485 return false; 486 } 487 try { 488 ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(eAction, coordAction, conf); 489 List<Element> elemList = eSla.getChildren(); 490 for (Element elem : elemList) { 491 String updated; 492 try { 493 updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim()); 494 } 495 catch (Exception e) { 496 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 497 } 498 elem.removeContent(); 499 elem.addContent(updated); 500 } 501 } 502 catch (Exception e) { 503 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 504 } 505 return true; 506 } 507 508 /** 509 * Materialize one instance for specific nominal time. It includes: 1. 510 * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize 511 * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and 512 * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag 513 * 514 * @param jobId coordinator job id 515 * @param dryrun true if it is dryrun 516 * @param eAction frequency unexploded-job 517 * @param nominalTime materialization time 518 * @param actualTime action actual time 519 * @param instanceCount instance numbers 520 * @param conf job configuration 521 * @param actionBean CoordinatorActionBean to materialize 522 * @return one materialized action for specific nominal time 523 * @throws Exception 524 */ 525 @SuppressWarnings("unchecked") 526 public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime, 527 Date actualTime, int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception { 528 String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + ""); 529 SyncCoordAction appInst = new SyncCoordAction(); 530 appInst.setActionId(actionId); 531 appInst.setName(eAction.getAttributeValue("name")); 532 appInst.setNominalTime(nominalTime); 533 appInst.setActualTime(actualTime); 534 String frequency = eAction.getAttributeValue("frequency"); 535 appInst.setFrequency(frequency); 536 appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit"))); 537 appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone"))); 538 appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration"))); 539 540 boolean isInputLogicSpecified = CoordUtils.isInputLogicSpecified(eAction); 541 542 Element inputList = eAction.getChild("input-events", eAction.getNamespace()); 543 List<Element> dataInList = null; 544 if (inputList != null) { 545 dataInList = inputList.getChildren("data-in", eAction.getNamespace()); 546 materializeInputDataEvents(dataInList, appInst, conf, actionBean, isInputLogicSpecified); 547 } 548 549 if(isInputLogicSpecified){ 550 evaluateInputCheck(eAction.getChild(CoordInputLogicEvaluator.INPUT_LOGIC, eAction.getNamespace()), 551 CoordELEvaluator.createDataEvaluator(eAction, conf, actionId)); 552 } 553 Element outputList = eAction.getChild("output-events", eAction.getNamespace()); 554 List<Element> dataOutList = null; 555 if (outputList != null) { 556 dataOutList = outputList.getChildren("data-out", eAction.getNamespace()); 557 materializeOutputDataEvents(dataOutList, appInst, conf); 558 } 559 560 eAction.removeAttribute("start"); 561 eAction.removeAttribute("end"); 562 eAction.setAttribute("instance-number", Integer.toString(instanceCount)); 563 eAction.setAttribute("action-nominal-time", DateUtils.formatDateOozieTZ(nominalTime)); 564 eAction.setAttribute("action-actual-time", DateUtils.formatDateOozieTZ(actualTime)); 565 566 // Setting up action bean 567 actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString()); 568 actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString()); 569 actionBean.setCreatedTime(actualTime); 570 actionBean.setJobId(jobId); 571 actionBean.setId(actionId); 572 actionBean.setLastModifiedTime(new Date()); 573 actionBean.setStatus(CoordinatorAction.Status.WAITING); 574 actionBean.setActionNumber(instanceCount); 575 actionBean.setNominalTime(nominalTime); 576 boolean isSla = CoordCommandUtils.materializeSLA(eAction, actionBean, conf); 577 if (isSla == true) { 578 actionBean.setSlaXml(XmlUtils.prettyPrint( 579 eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"))) 580 .toString()); 581 } 582 583 // actionBean.setTrackerUri(trackerUri);//TOOD: 584 // actionBean.setConsoleUrl(consoleUrl); //TODO: 585 // actionBean.setType(type);//TODO: 586 // actionBean.setErrorInfo(errorCode, errorMessage); //TODO: 587 // actionBean.setExternalStatus(externalStatus);//TODO 588 if (!dryrun) { 589 return XmlUtils.prettyPrint(eAction).toString(); 590 } 591 else { 592 return dryRunCoord(eAction, actionBean); 593 } 594 } 595 596 597 /** 598 * @param eAction the actionXml related element 599 * @param actionBean the coordinator action bean 600 * @return actionXml returns actionXml as String 601 * @throws Exception 602 */ 603 static String dryRunCoord(Element eAction, CoordinatorActionBean actionBean) throws Exception { 604 String action = XmlUtils.prettyPrint(eAction).toString(); 605 StringBuilder actionXml = new StringBuilder(action); 606 Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf())); 607 actionBean.setActionXml(action); 608 609 if (CoordUtils.isInputLogicSpecified(eAction)) { 610 new CoordInputLogicEvaluatorUtil(actionBean).validateInputLogic(); 611 } 612 613 boolean isPushDepAvailable = true; 614 String pushMissingDependencies = actionBean.getPushInputDependencies().getMissingDependencies(); 615 if (pushMissingDependencies != null) { 616 ActionDependency actionDependencies = DependencyChecker.checkForAvailability(pushMissingDependencies, 617 actionConf, true); 618 if (actionDependencies.getMissingDependencies().size() != 0) { 619 isPushDepAvailable = false; 620 } 621 622 } 623 boolean isPullDepAvailable = true; 624 CoordActionInputCheckXCommand coordActionInput = new CoordActionInputCheckXCommand(actionBean.getId(), 625 actionBean.getJobId()); 626 if (actionBean.getMissingDependencies() != null) { 627 StringBuilder existList = new StringBuilder(); 628 StringBuilder nonExistList = new StringBuilder(); 629 StringBuilder nonResolvedList = new StringBuilder(); 630 getResolvedList(actionBean.getPullInputDependencies().getMissingDependencies(), nonExistList, nonResolvedList); 631 isPullDepAvailable = actionBean.getPullInputDependencies().checkPullMissingDependencies(actionBean, 632 existList, nonExistList); 633 634 } 635 636 if (isPullDepAvailable && isPushDepAvailable) { 637 // Check for latest/future 638 boolean isLatestFutureDepAvailable = coordActionInput.checkUnResolvedInput(actionBean, actionXml, 639 actionConf); 640 if (isLatestFutureDepAvailable) { 641 String newActionXml = CoordActionInputCheckXCommand.resolveCoordConfiguration(actionXml, actionConf, 642 actionBean.getId()); 643 actionXml.replace(0, actionXml.length(), newActionXml); 644 } 645 } 646 647 return actionXml.toString(); 648 } 649 650 /** 651 * Materialize all <input-events>/<data-in> or <output-events>/<data-out> 652 * tags Create uris for resolved instances. Create unresolved instance for 653 * latest()/future(). 654 * 655 * @param events 656 * @param appInst 657 * @param conf 658 * @throws Exception 659 */ 660 private static void materializeOutputDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf) 661 throws Exception { 662 663 if (events == null) { 664 return; 665 } 666 667 for (Element event : events) { 668 StringBuilder instances = new StringBuilder(); 669 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf); 670 // Handle list of instance tag 671 resolveInstances(event, instances, appInst, conf, eval); 672 // Handle start-instance and end-instance 673 resolveInstanceRange(event, instances, appInst, conf, eval); 674 // Separate out the unresolved instances 675 separateResolvedAndUnresolved(event, instances); 676 677 } 678 } 679 680 private static void evaluateInputCheck(Element root, ELEvaluator evalInputLogic) throws Exception { 681 for (Object event : root.getChildren()) { 682 Element inputElement = (Element) event; 683 684 resolveAttribute("dataset", inputElement, evalInputLogic); 685 resolveAttribute("name", inputElement, evalInputLogic); 686 resolveAttribute("min", inputElement, evalInputLogic); 687 resolveAttribute("wait", inputElement, evalInputLogic); 688 if (!inputElement.getChildren().isEmpty()) { 689 evaluateInputCheck(inputElement, evalInputLogic); 690 } 691 } 692 } 693 694 private static String resolveAttribute(String attrName, Element elem, ELEvaluator eval) throws CoordinatorJobException { 695 Attribute attr = elem.getAttribute(attrName); 696 String val = null; 697 if (attr != null) { 698 try { 699 val = CoordELFunctions.evalAndWrap(eval, attr.getValue().trim()); 700 } 701 catch (Exception e) { 702 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 703 } 704 attr.setValue(val); 705 } 706 return val; 707 } 708 709 public static void materializeInputDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, 710 CoordinatorActionBean actionBean, boolean isInputLogicSpecified) throws Exception { 711 712 if (events == null) { 713 return; 714 } 715 CoordInputDependency coordPullInputDependency = CoordInputDependencyFactory 716 .createPullInputDependencies(isInputLogicSpecified); 717 CoordInputDependency coordPushInputDependency = CoordInputDependencyFactory 718 .createPushInputDependencies(isInputLogicSpecified); 719 List<Pair<String, String>> unresolvedList = new ArrayList<Pair<String, String>>(); 720 721 URIHandlerService uriService = Services.get().get(URIHandlerService.class); 722 723 for (Element event : events) { 724 StringBuilder instances = new StringBuilder(); 725 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf); 726 // Handle list of instance tag 727 resolveInstances(event, instances, appInst, conf, eval); 728 // Handle start-instance and end-instance 729 resolveInstanceRange(event, instances, appInst, conf, eval); 730 // Separate out the unresolved instances 731 String resolvedList = separateResolvedAndUnresolved(event, instances); 732 String name = event.getAttribute("name").getValue(); 733 734 if (!resolvedList.isEmpty()) { 735 Element uri = event.getChild("dataset", event.getNamespace()).getChild("uri-template", 736 event.getNamespace()); 737 738 String uriTemplate = uri.getText(); 739 URI baseURI = uriService.getAuthorityWithScheme(uriTemplate); 740 URIHandler handler = uriService.getURIHandler(baseURI); 741 List<CoordInputInstance> inputInstanceList = new ArrayList<CoordInputInstance>(); 742 743 for (String inputInstance : resolvedList.split("#")) { 744 inputInstanceList.add(new CoordInputInstance(inputInstance, false)); 745 } 746 747 if (handler.getDependencyType(baseURI).equals(DependencyType.PULL)) { 748 coordPullInputDependency.addInputInstanceList(name, inputInstanceList); 749 } 750 else { 751 coordPushInputDependency.addInputInstanceList(name, inputInstanceList); 752 753 } 754 } 755 756 String tmpUnresolved = event.getChildTextTrim(UNRESOLVED_INSTANCES_TAG, event.getNamespace()); 757 if (tmpUnresolved != null) { 758 unresolvedList.add(new Pair<String,String>(name, tmpUnresolved)); 759 } 760 } 761 for (Pair<String, String> unresolvedDataset : unresolvedList) { 762 coordPullInputDependency.addUnResolvedList(unresolvedDataset.getFirst(), unresolvedDataset.getSecond()); 763 } 764 actionBean.setPullInputDependencies(coordPullInputDependency); 765 actionBean.setPushInputDependencies(coordPushInputDependency); 766 actionBean.setMissingDependencies(coordPullInputDependency.serialize()); 767 actionBean.setPushMissingDependencies(coordPushInputDependency.serialize()); 768 769 } 770 /** 771 * Get resolved string from missDepList 772 * 773 * @param missDepList 774 * @param resolved 775 * @param unresolved 776 * @return resolved string 777 */ 778 public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) { 779 if (missDepList != null) { 780 int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR); 781 if (index < 0) { 782 resolved.append(missDepList); 783 } 784 else { 785 resolved.append(missDepList.substring(0, index)); 786 unresolved.append(missDepList.substring(index + RESOLVED_UNRESOLVED_SEPARATOR.length())); 787 } 788 } 789 return resolved.toString(); 790 } 791 792 /** 793 * Get the next action time after a given time 794 * 795 * @param targetDate 796 * @param coordJob 797 * @return the next valid action time 798 */ 799 public static Date getNextValidActionTimeForCronFrequency(Date targetDate, CoordinatorJobBean coordJob) throws ParseException { 800 801 String freq = coordJob.getFrequency(); 802 TimeZone tz = DateUtils.getOozieProcessingTimeZone(); 803 String[] cronArray = freq.split(" "); 804 Date nextTime = null; 805 806 // Current CronExpression doesn't support operations 807 // where both date of months and day of weeks are specified. 808 // As a result, we need to split this scenario into two cases 809 // and return the earlier time 810 if (!cronArray[2].trim().equals("?") && !cronArray[4].trim().equals("?")) { 811 812 // When any one of day of month or day of week fields is a wildcard 813 // we need to replace the wildcard with "?" 814 if (cronArray[2].trim().equals("*") || cronArray[4].trim().equals("*")) { 815 if (cronArray[2].trim().equals("*")) { 816 cronArray[2] = "?"; 817 } 818 else { 819 cronArray[4] = "?"; 820 } 821 freq= StringUtils.join(cronArray, " "); 822 823 // The cronExpression class takes second 824 // as the first field where oozie is operating on 825 // minute basis 826 CronExpression expr = new CronExpression("0 " + freq); 827 expr.setTimeZone(tz); 828 nextTime = expr.getNextValidTimeAfter(targetDate); 829 } 830 // If both fields are specified by non-wildcards, 831 // we need to split it into two expressions 832 else { 833 String[] cronArray1 = freq.split(" "); 834 String[] cronArray2 = freq.split(" "); 835 836 cronArray1[2] = "?"; 837 cronArray2[4] = "?"; 838 839 String freq1 = StringUtils.join(cronArray1, " "); 840 String freq2 = StringUtils.join(cronArray2, " "); 841 842 // The cronExpression class takes second 843 // as the first field where oozie is operating on 844 // minute basis 845 CronExpression expr1 = new CronExpression("0 " + freq1); 846 expr1.setTimeZone(tz); 847 CronExpression expr2 = new CronExpression("0 " + freq2); 848 expr2.setTimeZone(tz); 849 nextTime = expr1.getNextValidTimeAfter(targetDate); 850 Date nextTime2 = expr2.getNextValidTimeAfter(targetDate); 851 nextTime = nextTime.compareTo(nextTime2) < 0 ? nextTime: nextTime2; 852 } 853 } 854 else { 855 // The cronExpression class takes second 856 // as the first field where oozie is operating on 857 // minute basis 858 CronExpression expr = new CronExpression("0 " + freq); 859 expr.setTimeZone(tz); 860 nextTime = expr.getNextValidTimeAfter(targetDate); 861 } 862 863 return nextTime; 864 } 865 866 /** 867 * Computes the nominal time of the next action. 868 * Based on CoordMaterializeTransitionXCommand#materializeActions 869 * 870 * The Coordinator Job needs to have the frequency, time unit, time zone, start time, end time, and job xml. 871 * The Coordinator Action needs to have the nominal time and action number. 872 * 873 * @param coordJob The Coordinator Job 874 * @param coordAction The Coordinator Action 875 * @return the nominal time of the next action 876 * @throws ParseException 877 * @throws JDOMException 878 */ 879 public static Date computeNextNominalTime(CoordinatorJobBean coordJob, CoordinatorActionBean coordAction) 880 throws ParseException, JDOMException { 881 Date nextNominalTime; 882 boolean isCronFrequency = false; 883 int freq = -1; 884 try { 885 freq = Integer.parseInt(coordJob.getFrequency()); 886 } catch (NumberFormatException e) { 887 isCronFrequency = true; 888 } 889 890 if (isCronFrequency) { 891 nextNominalTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(coordAction.getNominalTime(), coordJob); 892 } else { 893 TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone()); 894 Calendar nextNominalTimeCal = Calendar.getInstance(appTz); 895 nextNominalTimeCal.setTime(coordJob.getStartTimestamp()); 896 TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr()); 897 // Action Number is indexed by 1, so no need to +1 here 898 nextNominalTimeCal.add(freqTU.getCalendarUnit(), coordAction.getActionNumber() * freq); 899 String jobXml = coordJob.getJobXml(); 900 Element eJob = XmlUtils.parseXml(jobXml); 901 TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration")); 902 // Move to the End of duration, if needed. 903 DateUtils.moveToEnd(nextNominalTimeCal, endOfFlag); 904 nextNominalTime = nextNominalTimeCal.getTime(); 905 } 906 907 // If the next nominal time is after the job's end time, then this is the last action, so return null 908 if (nextNominalTime.after(coordJob.getEndTime())) { 909 nextNominalTime = null; 910 } 911 return nextNominalTime; 912 } 913 914 public static boolean pathExists(String sPath, Configuration actionConf, String user) throws IOException, 915 URISyntaxException, URIHandlerException { 916 URI uri = new URI(sPath); 917 URIHandlerService service = Services.get().get(URIHandlerService.class); 918 URIHandler handler = service.getURIHandler(uri); 919 return handler.exists(uri, actionConf, user); 920 } 921 922 public static boolean pathExists(String sPath, Configuration actionConf) throws IOException, URISyntaxException, 923 URIHandlerException { 924 String user = ParamChecker.notEmpty(actionConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME); 925 return pathExists(sPath, actionConf, user); 926 } 927 928 public static String getFirstMissingDependency(CoordinatorActionBean coordAction) { 929 CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies(); 930 CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies(); 931 String firstMissingDependencies = coordPullInputDependency.getFirstMissingDependency(); 932 if (StringUtils.isEmpty(firstMissingDependencies) || firstMissingDependencies.trim().startsWith("${coord:")) { 933 firstMissingDependencies = coordPushInputDependency.getFirstMissingDependency(); 934 } 935 return firstMissingDependencies; 936 } 937 938 /** 939 * Class to find get start instance 940 */ 941 private static class StartInstanceFinder { 942 943 private int startIndex; 944 private TimeUnit timeUnit; 945 private TimeZone datasetTimeZone; 946 private Date nominalTime; 947 948 /** 949 * @param startIndex dataset index 950 * @param timeUnit 951 * @param datasetTimeZone 952 * @param nominalTime nominal time of action 953 */ 954 public StartInstanceFinder(int startIndex, TimeUnit timeUnit, TimeZone datasetTimeZone, Date nominalTime) { 955 this.startIndex = startIndex; 956 this.timeUnit = timeUnit; 957 this.datasetTimeZone = datasetTimeZone; 958 this.nominalTime = nominalTime; 959 } 960 961 /** 962 * Calculates the start instance. It put the start instance to the start 963 * of a day i.e. 00:00:00. 964 */ 965 public Calendar getStartInstance() throws Exception { 966 Calendar startInstance = Calendar.getInstance(datasetTimeZone); 967 startInstance.setTime(nominalTime); 968 startInstance.set(Calendar.HOUR_OF_DAY, 0); 969 startInstance.set(Calendar.MINUTE, 0); 970 startInstance.set(Calendar.SECOND, 0); 971 switch (timeUnit) { 972 case WEEK: 973 startInstance.set(Calendar.DAY_OF_WEEK, startInstance.getFirstDayOfWeek()); 974 startInstance.add(Calendar.WEEK_OF_YEAR, startIndex + 1); 975 break; 976 case MONTH: 977 int FIRST_DAY_OF_MONTH = 1; 978 startInstance.set(Calendar.DAY_OF_MONTH, FIRST_DAY_OF_MONTH); 979 startInstance.add(Calendar.MONTH, startIndex + 1); 980 break; 981 case DAY: 982 startInstance.add(Calendar.DATE, startIndex + 1); 983 break; 984 default: 985 } 986 return startInstance; 987 } 988 } 989 990}