Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.client;
020
021import java.io.BufferedReader;
022import java.io.File;
023import java.io.FileReader;
024import java.io.IOException;
025import java.io.InputStreamReader;
026import java.net.HttpURLConnection;
027import java.util.Properties;
028
029import org.apache.oozie.cli.OozieCLI;
030import org.apache.oozie.client.rest.JsonTags;
031import org.apache.oozie.client.rest.RestConstants;
032import org.json.simple.JSONObject;
033import org.json.simple.JSONValue;
034
035public class XOozieClient extends OozieClient {
036    public static final String RM = "yarn.resourcemanager.address";
037    public static final String NN = "fs.default.name";
038    public static final String NN_2 = "fs.defaultFS";
039
040    public static final String PIG_SCRIPT = "oozie.pig.script";
041
042    public static final String PIG_OPTIONS = "oozie.pig.options";
043
044    public static final String PIG_SCRIPT_PARAMS = "oozie.pig.script.params";
045
046    public static final String HIVE_SCRIPT = "oozie.hive.script";
047
048    public static final String HIVE_OPTIONS = "oozie.hive.options";
049
050    public static final String HIVE_SCRIPT_PARAMS = "oozie.hive.script.params";
051
052    public static final String SQOOP_COMMAND = "oozie.sqoop.command";
053
054    public static final String SQOOP_OPTIONS = "oozie.sqoop.options";
055
056    public static final String FILES = "oozie.files";
057
058    public static final String ARCHIVES = "oozie.archives";
059
060    public static final String IS_PROXY_SUBMISSION = "oozie.proxysubmission";
061
062    protected XOozieClient() {
063    }
064
065    /**
066     * Create an eXtended Workflow client instance.
067     *
068     * @param oozieUrl URL of the Oozie instance it will interact with.
069     */
070    public XOozieClient(String oozieUrl) {
071        super(oozieUrl);
072    }
073
074    private String readScript(String script) throws IOException {
075        if (!new File(script).exists()) {
076            throw new IOException("Error: script file [" + script + "] does not exist");
077        }
078
079        BufferedReader br = null;
080        try {
081            br = new BufferedReader(new FileReader(script));
082            StringBuilder sb = new StringBuilder();
083            String line;
084            while ((line = br.readLine()) != null) {
085                sb.append(line + "\n");
086            }
087            return sb.toString();
088        }
089        finally {
090            try {
091                br.close();
092            }
093            catch (IOException ex) {
094                System.err.println("Error: " + ex.getMessage());
095            }
096        }
097    }
098
099    private String serializeSqoopCommand(String[] command) {
100        StringBuilder sb = new StringBuilder();
101        for (String arg : command) {
102            sb.append(arg).append("\n");
103        }
104        return sb.toString();
105    }
106
107    static void setStrings(Properties conf, String key, String[] values) {
108        if (values != null) {
109            conf.setProperty(key + ".size", (new Integer(values.length)).toString());
110            for (int i = 0; i < values.length; i++) {
111                conf.setProperty(key + "." + i, values[i]);
112            }
113        }
114    }
115
116    private void validateHttpSubmitConf(Properties conf) {
117        String RM = conf.getProperty(XOozieClient.RM);
118        if (RM == null) {
119            throw new RuntimeException("Resource manager is not specified in conf");
120        }
121
122        String NN = conf.getProperty(XOozieClient.NN);
123        String NN_2 = conf.getProperty(XOozieClient.NN_2);
124        if (NN == null) {
125            if(NN_2 == null) {
126                throw new RuntimeException("namenode is not specified in conf");
127            } else {
128                NN = NN_2;
129            }
130        }
131
132        String libPath = conf.getProperty(LIBPATH);
133        if (libPath == null) {
134            throw new RuntimeException("libpath is not specified in conf");
135        }
136        if (!libPath.contains(":/")) {
137            String newLibPath;
138            if (libPath.startsWith("/")) {
139                if(NN.endsWith("/")) {
140                    newLibPath = NN + libPath.substring(1);
141                } else {
142                    newLibPath = NN + libPath;
143                }
144            } else {
145                throw new RuntimeException("libpath should be absolute");
146            }
147            conf.setProperty(LIBPATH, newLibPath);
148        }
149
150        conf.setProperty(IS_PROXY_SUBMISSION, "true");
151    }
152
153    /**
154     * Submit a Pig job via HTTP.
155     *
156     * @param conf job configuration.
157     * @param pigScriptFile pig script file.
158     * @param pigArgs pig arguments string.
159     * @return the job Id.
160     * @throws java.io.IOException thrown if there is a problem with pig script file.
161     * @throws OozieClientException thrown if the job could not be submitted.
162     */
163    @Deprecated
164    public String submitPig(Properties conf, String pigScriptFile, String[] pigArgs) throws IOException, OozieClientException {
165        return submitScriptLanguage(conf, pigScriptFile, pigArgs, OozieCLI.PIG_CMD);
166    }
167
168    /**
169     * Submit a Pig or Hive job via HTTP.
170     *
171     * @param conf job configuration.
172     * @param scriptFile  script file.
173     * @param args  arguments string.
174     * @param jobType job type.
175     * @return the job Id.
176     * @throws java.io.IOException thrown if there is a problem with script file.
177     * @throws OozieClientException thrown if the job could not be submitted.
178     */
179    public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String jobType)
180            throws IOException, OozieClientException {
181        return submitScriptLanguage(conf, scriptFile, args, null, jobType);
182    }
183
184    /**
185     * Submit a Pig or Hive job via HTTP.
186     *
187     * @param conf job configuration.
188     * @param scriptFile  script file.
189     * @param args  arguments string.
190     * @param params parameters string.
191     * @param jobType job type.
192     * @return the job Id.
193     * @throws java.io.IOException thrown if there is a problem with file.
194     * @throws OozieClientException thrown if the job could not be submitted.
195     */
196    public String submitScriptLanguage(Properties conf, String scriptFile, String[] args, String[] params,
197        String jobType)
198            throws IOException, OozieClientException {
199        OozieClient.notNull(conf, "conf");
200        OozieClient.notNull(scriptFile, "scriptFile");
201        validateHttpSubmitConf(conf);
202
203        String script = "";
204        String options = "";
205        String scriptParams = "";
206
207        if (jobType.equals(OozieCLI.HIVE_CMD)) {
208            script = XOozieClient.HIVE_SCRIPT;
209            options = XOozieClient.HIVE_OPTIONS;
210            scriptParams = XOozieClient.HIVE_SCRIPT_PARAMS;
211        }
212        else if (jobType.equals(OozieCLI.PIG_CMD)) {
213            script =  XOozieClient.PIG_SCRIPT;
214            options = XOozieClient.PIG_OPTIONS;
215            scriptParams = XOozieClient.PIG_SCRIPT_PARAMS;
216        }
217        else {
218            throw new IllegalArgumentException("jobType must be either pig or hive");
219        }
220
221        conf.setProperty(script, readScript(scriptFile));
222        setStrings(conf, options, args);
223        setStrings(conf, scriptParams, params);
224
225        return (new HttpJobSubmit(conf, jobType)).call();
226    }
227
228    /**
229     * Submit a Sqoop job via HTTP.
230     *
231     * @param conf job configuration.
232     * @param command sqoop command to run.
233     * @param args  arguments string.
234     * @return the job Id.
235     * @throws OozieClientException thrown if the job could not be submitted.
236     */
237    public String submitSqoop(Properties conf, String[] command, String[] args)
238            throws OozieClientException {
239        OozieClient.notNull(conf, "conf");
240        OozieClient.notNull(command, "command");
241        validateHttpSubmitConf(conf);
242
243        conf.setProperty(XOozieClient.SQOOP_COMMAND, serializeSqoopCommand(command));
244        setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
245
246        return (new HttpJobSubmit(conf, OozieCLI.SQOOP_CMD)).call();
247    }
248
249    /**
250     * Submit a Map/Reduce job via HTTP.
251     *
252     * @param conf job configuration.
253     * @return the job Id.
254     * @throws OozieClientException thrown if the job could not be submitted.
255     */
256    public String submitMapReduce(Properties conf) throws OozieClientException {
257        OozieClient.notNull(conf, "conf");
258        validateHttpSubmitConf(conf);
259
260        return (new HttpJobSubmit(conf, "mapreduce")).call();
261    }
262
263    private class HttpJobSubmit extends ClientCallable<String> {
264        private Properties conf;
265
266        HttpJobSubmit(Properties conf, String jobType) {
267            super("POST", RestConstants.JOBS, "", prepareParams(RestConstants.JOBTYPE_PARAM, jobType));
268            this.conf = notNull(conf, "conf");
269        }
270
271        @Override
272        protected String call(HttpURLConnection conn) throws IOException, OozieClientException {
273            conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE);
274            writeToXml(conf, conn.getOutputStream());
275            if (conn.getResponseCode() == HttpURLConnection.HTTP_CREATED) {
276                JSONObject json = (JSONObject) JSONValue.parse(new InputStreamReader(conn.getInputStream()));
277                return (String) json.get(JsonTags.JOB_ID);
278            }
279            if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
280                handleError(conn);
281            }
282            return null;
283        }
284    }
285
286    /**
287     * set LIBPATH for HTTP submission job.
288     *
289     * @param conf Configuration object.
290     * @param pathStr lib HDFS path.
291     */
292    public void setLib(Properties conf, String pathStr) {
293        conf.setProperty(LIBPATH, pathStr);
294    }
295
296    /**
297     * The equivalent to &lt;file&gt; tag in oozie's workflow xml.
298     *
299     * @param conf Configuration object.
300     * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
301     *             For example, "/user/oozie/parameter_file#myparams". If no "#..." is specified, file name will be used as
302     *             symbolic link name.
303     */
304    public void addFile(Properties conf, String file) {
305        OozieClient.notEmpty(file, "file");
306        String files = conf.getProperty(FILES);
307        conf.setProperty(FILES, files == null ? file : files + "," + file);
308    }
309
310    /**
311     * The equivalent to &lt;archive&gt; tag in oozie's workflow xml.
312     *
313     * @param conf Configuration object.
314     * @param file file HDFS path. A "#..." symbolic string can be appended to the path to specify symbolic link name.
315     *             For example, "/user/oozie/udf1.jar#my.jar". If no "#..." is specified, file name will be used as
316     *             symbolic link name.
317     */
318    public void addArchive(Properties conf, String file) {
319        OozieClient.notEmpty(file, "file");
320        String files = conf.getProperty(ARCHIVES);
321        conf.setProperty(ARCHIVES, files == null ? file : files + "," + file);
322    }
323}