Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.action.hadoop;
020
021import java.io.BufferedReader;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.InputStreamReader;
025import java.math.BigInteger;
026import java.security.MessageDigest;
027import java.security.NoSuchAlgorithmException;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.Properties;
035
036import org.apache.hadoop.conf.Configuration;
037import org.apache.hadoop.fs.FileSystem;
038import org.apache.hadoop.fs.Path;
039import org.apache.hadoop.io.SequenceFile;
040import org.apache.hadoop.io.Text;
041import org.apache.hadoop.mapred.Counters;
042import org.apache.hadoop.mapred.RunningJob;
043import org.apache.hadoop.security.UserGroupInformation;
044import org.apache.oozie.client.OozieClient;
045import org.apache.oozie.client.WorkflowAction;
046import org.apache.oozie.service.HadoopAccessorException;
047import org.apache.oozie.service.HadoopAccessorService;
048import org.apache.oozie.service.Services;
049import org.apache.oozie.service.URIHandlerService;
050import org.apache.oozie.service.UserGroupInformationService;
051import org.apache.oozie.util.IOUtils;
052import org.apache.oozie.util.PropertiesUtils;
053
054public class LauncherHelper {
055
056    public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag";
057
058    public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
059            throws HadoopAccessorException, IOException {
060        String jobId = null;
061        Path recoveryFile = new Path(actionDir, recoveryId);
062        FileSystem fs = Services.get().get(HadoopAccessorService.class)
063                .createFileSystem(launcherConf.get("user.name"),recoveryFile.toUri(), launcherConf);
064
065        if (fs.exists(recoveryFile)) {
066            InputStream is = fs.open(recoveryFile);
067            BufferedReader reader = new BufferedReader(new InputStreamReader(is));
068            jobId = reader.readLine();
069            reader.close();
070        }
071        return jobId;
072
073    }
074
075    public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
076        // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via
077        // <configuration> property
078        if (javaMainClass != null && !javaMainClass.equals("")) {
079            launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
080        }
081    }
082
083    public static void setupLauncherURIHandlerConf(Configuration launcherConf) {
084        for(Map.Entry<String, String> entry : Services.get().get(URIHandlerService.class).getLauncherConfig()) {
085            launcherConf.set(entry.getKey(), entry.getValue());
086        }
087    }
088
089    public static void setupMainArguments(Configuration launcherConf, String[] args) {
090        launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
091        for (int i = 0; i < args.length; i++) {
092            launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
093        }
094    }
095
096    public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
097        launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
098    }
099
100    /**
101     * Set the maximum value of stats data
102     *
103     * @param launcherConf the oozie launcher configuration
104     * @param maxStatsData the maximum allowed size of stats data
105     */
106    public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){
107        launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
108    }
109
110    /**
111     * Set the maximum number of globbed files/dirs
112     *
113     * @param launcherConf the oozie launcher configuration
114     * @param fsGlobMax the maximum number of files/dirs for FS operation
115     */
116    public static void setupMaxFSGlob(Configuration launcherConf, int fsGlobMax){
117        launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
118    }
119
120    public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir,
121            String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
122
123        launcherConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId);
124        launcherConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId);
125        launcherConf.set(LauncherAMUtils.OOZIE_ACTION_DIR_PATH, actionDir.toString());
126        launcherConf.set(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID, recoveryId);
127        launcherConf.set(LauncherAMUtils.ACTION_PREPARE_XML, prepareXML);
128
129        actionConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId);
130        actionConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId);
131
132        if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
133          List<String> purgedEntries = new ArrayList<String>();
134          Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
135          for (String entry : entries) {
136            if (entry.contains("#")) {
137              purgedEntries.add(entry);
138            }
139          }
140          actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
141          launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
142        }
143    }
144
145    public static void setupYarnRestartHandling(Configuration launcherJobConf, Configuration actionConf, String launcherTag,
146                                                long launcherTime)
147            throws NoSuchAlgorithmException {
148        launcherJobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, launcherTime);
149        // Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length)
150        String tag = getTag(launcherTag);
151        // keeping the oozie.child.mapreduce.job.tags instead of mapreduce.job.tags to avoid killing launcher itself.
152        // mapreduce.job.tags should only go to child job launch by launcher.
153        actionConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, tag);
154    }
155
156    public static String getTag(String launcherTag) throws NoSuchAlgorithmException {
157        MessageDigest digest = MessageDigest.getInstance("MD5");
158        digest.update(launcherTag.getBytes(), 0, launcherTag.length());
159        String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16);
160        return md5;
161    }
162
163    public static boolean isMainDone(RunningJob runningJob) throws IOException {
164        return runningJob.isComplete();
165    }
166
167    public static boolean isMainSuccessful(RunningJob runningJob) throws IOException {
168        boolean succeeded = runningJob.isSuccessful();
169        if (succeeded) {
170            Counters counters = runningJob.getCounters();
171            if (counters != null) {
172                Counters.Group group = counters.getGroup(LauncherAMUtils.COUNTER_GROUP);
173                if (group != null) {
174                    succeeded = group.getCounter(LauncherAMUtils.COUNTER_LAUNCHER_ERROR) == 0;
175                }
176            }
177        }
178        return succeeded;
179    }
180
181    /**
182     * Determine whether action has external child jobs or not
183     * @param actionData
184     * @return true/false
185     * @throws IOException
186     */
187    public static boolean hasExternalChildJobs(Map<String, String> actionData) throws IOException {
188        return actionData.containsKey(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS);
189    }
190
191    /**
192     * Determine whether action has output data or not
193     * @param actionData
194     * @return true/false
195     * @throws IOException
196     */
197    public static boolean hasOutputData(Map<String, String> actionData) throws IOException {
198        return actionData.containsKey(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS);
199    }
200
201    /**
202     * Determine whether action has external stats or not
203     * @param actionData
204     * @return true/false
205     * @throws IOException
206     */
207    public static boolean hasStatsData(Map<String, String> actionData) throws IOException{
208        return actionData.containsKey(LauncherAMUtils.ACTION_DATA_STATS);
209    }
210
211    /**
212     * Determine whether action has new id (id swap) or not
213     * @param actionData
214     * @return true/false
215     * @throws IOException
216     */
217    public static boolean hasIdSwap(Map<String, String> actionData) throws IOException {
218        return actionData.containsKey(LauncherAMUtils.ACTION_DATA_NEW_ID);
219    }
220
221    /**
222     * Get the sequence file path storing all action data
223     * @param actionDir
224     * @return Path returns sequence file path storing all action data
225     */
226    public static Path getActionDataSequenceFilePath(Path actionDir) {
227        return new Path(actionDir, LauncherAMUtils.ACTION_DATA_SEQUENCE_FILE);
228    }
229
230    /**
231     * Utility function to load the contents of action data sequence file into
232     * memory object
233     *
234     * @param fs Action Filesystem
235     * @param actionDir Path
236     * @param conf Configuration
237     * @return Map action data
238     * @throws IOException
239     * @throws InterruptedException
240     */
241    public static Map<String, String> getActionData(final FileSystem fs, final Path actionDir, final Configuration conf)
242            throws IOException, InterruptedException {
243        UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class);
244        UserGroupInformation ugi = ugiService.getProxyUser(conf.get(OozieClient.USER_NAME));
245
246        return ugi.doAs(new PrivilegedExceptionAction<Map<String, String>>() {
247            @Override
248            public Map<String, String> run() throws IOException {
249                Map<String, String> ret = new HashMap<String, String>();
250                Path seqFilePath = getActionDataSequenceFilePath(actionDir);
251                if (fs.exists(seqFilePath)) {
252                    SequenceFile.Reader seqFile = new SequenceFile.Reader(fs, seqFilePath, conf);
253                    Text key = new Text(), value = new Text();
254                    while (seqFile.next(key, value)) {
255                        ret.put(key.toString(), value.toString());
256                    }
257                    seqFile.close();
258                }
259                else { // maintain backward-compatibility. to be deprecated
260                    org.apache.hadoop.fs.FileStatus[] files = fs.listStatus(actionDir);
261                    InputStream is;
262                    BufferedReader reader = null;
263                    Properties props;
264                    if (files != null && files.length > 0) {
265                        for (int x = 0; x < files.length; x++) {
266                            Path file = files[x].getPath();
267                            if (file.equals(new Path(actionDir, "externalChildIds.properties"))) {
268                                is = fs.open(file);
269                                reader = new BufferedReader(new InputStreamReader(is));
270                                ret.put(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS,
271                                        IOUtils.getReaderAsString(reader, -1));
272                            }
273                            else if (file.equals(new Path(actionDir, "newId.properties"))) {
274                                is = fs.open(file);
275                                reader = new BufferedReader(new InputStreamReader(is));
276                                props = PropertiesUtils.readProperties(reader, -1);
277                                ret.put(LauncherAMUtils.ACTION_DATA_NEW_ID, props.getProperty("id"));
278                            }
279                            else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS))) {
280                                int maxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
281                                        2 * 1024);
282                                is = fs.open(file);
283                                reader = new BufferedReader(new InputStreamReader(is));
284                                ret.put(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils
285                                        .propertiesToString(PropertiesUtils.readProperties(reader, maxOutputData)));
286                            }
287                            else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_STATS))) {
288                                int statsMaxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
289                                        Integer.MAX_VALUE);
290                                is = fs.open(file);
291                                reader = new BufferedReader(new InputStreamReader(is));
292                                ret.put(LauncherAMUtils.ACTION_DATA_STATS, PropertiesUtils
293                                        .propertiesToString(PropertiesUtils.readProperties(reader, statsMaxOutputData)));
294                            }
295                            else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_ERROR_PROPS))) {
296                                is = fs.open(file);
297                                reader = new BufferedReader(new InputStreamReader(is));
298                                ret.put(LauncherAMUtils.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1));
299                            }
300                        }
301                    }
302                }
303                return ret;
304            }
305        });
306    }
307
308    public static String getActionYarnTag(Configuration conf, String parentId, WorkflowAction wfAction) {
309        String tag;
310        if ( conf != null && conf.get(OOZIE_ACTION_YARN_TAG) != null) {
311            tag = conf.get(OOZIE_ACTION_YARN_TAG) + "@" + wfAction.getName();
312        } else if (parentId != null) {
313            tag = parentId + "@" + wfAction.getName();
314        } else {
315            tag = wfAction.getId();
316        }
317        return tag;
318    }
319
320}