Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.command.coord;
020
021import org.apache.hadoop.conf.Configuration;
022import org.apache.oozie.action.hadoop.OozieJobInfo;
023import org.apache.oozie.client.CoordinatorAction;
024import org.apache.oozie.client.OozieClient;
025import org.apache.oozie.CoordinatorActionBean;
026import org.apache.oozie.DagEngineException;
027import org.apache.oozie.DagEngine;
028import org.apache.oozie.ErrorCode;
029import org.apache.oozie.SLAEventBean;
030import org.apache.oozie.WorkflowJobBean;
031import org.apache.oozie.command.CommandException;
032import org.apache.oozie.command.PreconditionException;
033import org.apache.oozie.service.DagEngineService;
034import org.apache.oozie.service.EventHandlerService;
035import org.apache.oozie.service.JPAService;
036import org.apache.oozie.service.Services;
037import org.apache.oozie.util.JobUtils;
038import org.apache.oozie.util.LogUtils;
039import org.apache.oozie.util.ParamChecker;
040import org.apache.oozie.util.XLog;
041import org.apache.oozie.util.XmlUtils;
042import org.apache.oozie.util.XConfiguration;
043import org.apache.oozie.util.db.SLADbOperations;
044import org.apache.oozie.client.SLAEvent.SlaAppType;
045import org.apache.oozie.client.SLAEvent.Status;
046import org.apache.oozie.client.rest.JsonBean;
047import org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry;
048import org.apache.oozie.executor.jpa.BatchQueryExecutor;
049import org.apache.oozie.executor.jpa.CoordActionQueryExecutor.CoordActionQuery;
050import org.apache.oozie.executor.jpa.JPAExecutorException;
051import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor;
052import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery;
053import org.jdom.Element;
054import org.jdom.JDOMException;
055
056import java.io.IOException;
057import java.io.StringReader;
058import java.util.ArrayList;
059import java.util.Date;
060import java.util.List;
061
062@SuppressWarnings("deprecation")
063public class CoordActionStartXCommand extends CoordinatorXCommand<Void> {
064
065    public static final String EL_ERROR = "EL_ERROR";
066    public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
067    public static final String COULD_NOT_START = "COULD_NOT_START";
068    public static final String START_DATA_MISSING = "START_DATA_MISSING";
069    public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
070    public static final String OOZIE_COORD_ACTION_NOMINAL_TIME = "oozie.coord.action.nominal_time";
071
072    private final XLog log = getLog();
073    private String actionId = null;
074    private String user = null;
075    private String appName = null;
076    private CoordinatorActionBean coordAction = null;
077    private JPAService jpaService = null;
078    private String jobId = null;
079    private List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
080    private List<JsonBean> insertList = new ArrayList<JsonBean>();
081
082    public CoordActionStartXCommand(String id, String user, String appName, String jobId) {
083        //super("coord_action_start", "coord_action_start", 1, XLog.OPS);
084        super("coord_action_start", "coord_action_start", 1);
085        this.actionId = ParamChecker.notEmpty(id, "id");
086        this.user = ParamChecker.notEmpty(user, "user");
087        this.appName = ParamChecker.notEmpty(appName, "appName");
088        this.jobId = jobId;
089    }
090
091    @Override
092    protected void setLogInfo() {
093        LogUtils.setLogInfo(actionId);
094    }
095
096    /**
097     * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
098     * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
099     * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
100     * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
101     * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
102     * Merge Action createdConf with actionXml to create new runConf with replaced variables
103     *
104     * @param action CoordinatorActionBean
105     * @return Configuration
106     * @throws CommandException
107     */
108    private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
109        String createdConf = action.getCreatedConf();
110        String actionXml = action.getActionXml();
111        Element workflowProperties = null;
112        try {
113            workflowProperties = XmlUtils.parseXml(actionXml);
114        }
115        catch (JDOMException e1) {
116            log.warn("Configuration parse error in:" + actionXml);
117            throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
118        }
119        // generate the 'runConf' for this action
120        // Step 1: runConf = createdConf
121        Configuration runConf = null;
122        try {
123            runConf = new XConfiguration(new StringReader(createdConf));
124
125        }
126        catch (IOException e1) {
127            log.warn("Configuration parse error in:" + createdConf);
128            throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
129        }
130        // Step 2: Merge local properties into runConf
131        // extract 'property' tags under 'configuration' block in the
132        // coordinator.xml (saved in actionxml column)
133        // convert Element to XConfiguration
134        Element configElement = workflowProperties.getChild("action", workflowProperties.getNamespace())
135                .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
136                                                                                  workflowProperties.getNamespace());
137        if (configElement != null) {
138            String strConfig = XmlUtils.prettyPrint(configElement).toString();
139            Configuration localConf;
140            try {
141                localConf = new XConfiguration(new StringReader(strConfig));
142            }
143            catch (IOException e1) {
144                log.warn("Configuration parse error in:" + strConfig);
145                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
146            }
147
148            // copy configuration properties in coordinator.xml to the runConf
149            XConfiguration.copy(localConf, runConf);
150        }
151
152        // Step 3: Extract value of 'app-path' in actionxml, and save it as a
153        // new property called 'oozie.wf.application.path'
154        // WF Engine requires the path to the workflow.xml to be saved under
155        // this property name
156        String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace())
157                .getChild("workflow", workflowProperties.getNamespace()).getChild("app-path",
158                        workflowProperties.getNamespace()).getValue();
159
160        // Copying application path in runconf.
161        runConf.set("oozie.wf.application.path", appPath);
162
163        // Step 4: Extract the runconf and copy the rerun config to runconf.
164        if (runConf.get(CoordRerunXCommand.RERUN_CONF) != null) {
165            Configuration rerunConf = null;
166            try {
167                rerunConf = new XConfiguration(new StringReader(runConf.get(CoordRerunXCommand.RERUN_CONF)));
168                XConfiguration.copy(rerunConf, runConf);
169            } catch (IOException e) {
170                log.warn("Configuration parse error in:" + rerunConf);
171                throw new CommandException(ErrorCode.E1005, e.getMessage(), e);
172            }
173            runConf.unset(CoordRerunXCommand.RERUN_CONF);
174        }
175        return runConf;
176    }
177
178    @Override
179    protected Void execute() throws CommandException {
180        boolean makeFail = true;
181        String errCode = "";
182        String errMsg = "";
183        ParamChecker.notEmpty(user, "user");
184
185        log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
186        if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
187            // log.debug("getting.. job id: " + coordAction.getJobId());
188            // create merged runConf to pass to WF Engine
189            Configuration runConf = mergeConfig(coordAction);
190            coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
191            // log.debug("%%% merged runconf=" +
192            // XmlUtils.prettyPrint(runConf).toString());
193            DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
194            try {
195                Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
196                SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(),
197                        Status.STARTED,
198                        SlaAppType.COORDINATOR_ACTION, log);
199                if(slaEvent != null) {
200                    insertList.add(slaEvent);
201                }
202                if (OozieJobInfo.isJobInfoEnabled()) {
203                    conf.set(OozieJobInfo.COORD_ID, actionId);
204                    conf.set(OozieJobInfo.COORD_NAME, appName);
205                    conf.set(OozieJobInfo.COORD_NOMINAL_TIME, coordAction.getNominalTimestamp().toString());
206                }
207                // Normalize workflow appPath here;
208                JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
209                if (coordAction.getExternalId() != null) {
210                    conf.setBoolean(OozieClient.RERUN_FAIL_NODES, true);
211                    dagEngine.reRun(coordAction.getExternalId(), conf);
212                } else {
213                    // Pushing the nominal time in conf to use for launcher tag search
214                    conf.set(OOZIE_COORD_ACTION_NOMINAL_TIME,String.valueOf(coordAction.getNominalTime().getTime()));
215                    String wfId = dagEngine.submitJobFromCoordinator(conf, actionId);
216                    coordAction.setExternalId(wfId);
217                }
218                coordAction.setStatus(CoordinatorAction.Status.RUNNING);
219                coordAction.incrementAndGetPending();
220
221                //store.updateCoordinatorAction(coordAction);
222                JPAService jpaService = Services.get().get(JPAService.class);
223                if (jpaService != null) {
224                    log.debug("Updating WF record for WFID :" + coordAction.getExternalId() + " with parent id: " + actionId);
225                    WorkflowJobBean wfJob = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW_STARTTIME,
226                            coordAction.getExternalId());
227                    wfJob.setParentId(actionId);
228                    wfJob.setLastModifiedTime(new Date());
229                    BatchQueryExecutor executor = BatchQueryExecutor.getInstance();
230                    updateList.add(new UpdateEntry<WorkflowJobQuery>(
231                            WorkflowJobQuery.UPDATE_WORKFLOW_PARENT_MODIFIED, wfJob));
232                    updateList.add(new UpdateEntry<CoordActionQuery>(
233                            CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
234                    try {
235                        executor.executeBatchInsertUpdateDelete(insertList, updateList, null);
236                        queue(new CoordActionNotificationXCommand(coordAction), 100);
237                        if (EventHandlerService.isEnabled()) {
238                            generateEvent(coordAction, user, appName, wfJob.getStartTime());
239                        }
240                    }
241                    catch (JPAExecutorException je) {
242                        throw new CommandException(je);
243                    }
244                }
245                else {
246                    log.error(ErrorCode.E0610);
247                }
248
249                makeFail = false;
250            }
251            catch (DagEngineException dee) {
252                errMsg = dee.getMessage();
253                errCode = dee.getErrorCode().toString();
254                log.warn("can not create DagEngine for submitting jobs", dee);
255            }
256            catch (CommandException ce) {
257                errMsg = ce.getMessage();
258                errCode = ce.getErrorCode().toString();
259                log.warn("command exception occurred ", ce);
260            }
261            catch (java.io.IOException ioe) {
262                errMsg = ioe.getMessage();
263                errCode = "E1005";
264                log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
265            }
266            catch (Exception ex) {
267                errMsg = ex.getMessage();
268                errCode = "E1005";
269                log.warn("can not create DagEngine for submitting jobs", ex);
270            }
271            finally {
272                if (makeFail == true) { // No DB exception occurs
273                    log.error("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
274                    coordAction.setStatus(CoordinatorAction.Status.FAILED);
275                    if (errMsg.length() > 254) { // Because table column size is 255
276                        errMsg = errMsg.substring(0, 255);
277                    }
278                    coordAction.setErrorMessage(errMsg);
279                    coordAction.setErrorCode(errCode);
280
281                    updateList = new ArrayList<UpdateEntry>();
282                    updateList.add(new UpdateEntry<CoordActionQuery>(
283                                    CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
284                    insertList = new ArrayList<JsonBean>();
285
286                    SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(),
287                            Status.FAILED,
288                            SlaAppType.COORDINATOR_ACTION, log);
289                    if(slaEvent != null) {
290                        insertList.add(slaEvent); //Update SLA events
291                    }
292                    try {
293                        // call JPAExecutor to do the bulk writes
294                        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
295                        if (EventHandlerService.isEnabled()) {
296                            generateEvent(coordAction, user, appName, null);
297                        }
298                    }
299                    catch (JPAExecutorException je) {
300                        throw new CommandException(je);
301                    }
302                    queue(new CoordActionReadyXCommand(coordAction.getJobId()));
303                }
304            }
305        }
306        return null;
307    }
308
309    @Override
310    public String getEntityKey() {
311        return this.jobId;
312    }
313
314    @Override
315    protected boolean isLockRequired() {
316        return true;
317    }
318
319    @Override
320    protected void loadState() throws CommandException {
321        jpaService = Services.get().get(JPAService.class);
322        try {
323            coordAction = jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionGetForStartJPAExecutor(
324                    actionId));
325        }
326        catch (JPAExecutorException je) {
327            throw new CommandException(je);
328        }
329        LogUtils.setLogInfo(coordAction);
330    }
331
332    @Override
333    protected void verifyPrecondition() throws PreconditionException {
334        if (coordAction.getStatus() != CoordinatorAction.Status.SUBMITTED) {
335            throw new PreconditionException(ErrorCode.E1100, "The coord action [" + actionId + "] must have status "
336                    + CoordinatorAction.Status.SUBMITTED.name() + " but has status [" + coordAction.getStatus().name() + "]");
337        }
338    }
339
340    @Override
341    public String getKey(){
342        return getName() + "_" + actionId;
343    }
344}