Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019
020package org.apache.oozie.sla.listener;
021
022import java.util.ArrayList;
023import java.util.HashSet;
024import java.util.List;
025import java.util.Properties;
026import java.util.Set;
027import java.util.concurrent.TimeUnit;
028import java.util.concurrent.atomic.AtomicInteger;
029
030import javax.mail.Address;
031import javax.mail.Message;
032import javax.mail.MessagingException;
033import javax.mail.NoSuchProviderException;
034import javax.mail.SendFailedException;
035import javax.mail.Session;
036import javax.mail.Transport;
037import javax.mail.internet.AddressException;
038import javax.mail.internet.InternetAddress;
039import javax.mail.internet.MimeMessage;
040import javax.mail.internet.MimeMessage.RecipientType;
041
042import org.apache.hadoop.conf.Configuration;
043import org.apache.oozie.action.email.EmailActionExecutor;
044import org.apache.oozie.action.email.EmailActionExecutor.JavaMailAuthenticator;
045import org.apache.oozie.client.event.SLAEvent;
046import org.apache.oozie.service.ConfigurationService;
047import org.apache.oozie.sla.listener.SLAEventListener;
048import org.apache.oozie.sla.service.SLAService;
049import org.apache.oozie.util.XLog;
050
051import com.google.common.annotations.VisibleForTesting;
052import com.google.common.cache.CacheBuilder;
053import com.google.common.cache.CacheLoader;
054import com.google.common.cache.LoadingCache;
055
056public class SLAEmailEventListener extends SLAEventListener {
057
058    public static final String SMTP_CONNECTION_TIMEOUT = EmailActionExecutor.CONF_PREFIX + "smtp.connectiontimeout";
059    public static final String SMTP_TIMEOUT = EmailActionExecutor.CONF_PREFIX + "smtp.timeout";
060    public static final String BLACKLIST_CACHE_TIMEOUT = EmailActionExecutor.CONF_PREFIX + "blacklist.cachetimeout";
061    public static final String BLACKLIST_FAIL_COUNT = EmailActionExecutor.CONF_PREFIX + "blacklist.failcount";
062    public static final String OOZIE_BASE_URL = "oozie.base.url";
063    private Session session;
064    private String oozieBaseUrl;
065    private InternetAddress fromAddr;
066    private String ADDRESS_SEPARATOR = ",";
067    private LoadingCache<String, AtomicInteger> blackList;
068    private int blacklistFailCount;
069    private final String BLACKLIST_CACHE_TIMEOUT_DEFAULT = "1800"; // in sec. default to 30 min
070    private final String BLACKLIST_FAIL_COUNT_DEFAULT = "2"; // stop sending when fail count equals or exceeds
071    private final String SMTP_HOST_DEFAULT = "localhost";
072    private final String SMTP_PORT_DEFAULT = "25";
073    private final boolean SMTP_AUTH_DEFAULT = false;
074    private final String SMTP_SOURCE_DEFAULT = "oozie@localhost";
075    private final String SMTP_CONNECTION_TIMEOUT_DEFAULT = "5000";
076    private final String SMTP_TIMEOUT_DEFAULT = "5000";
077    private static XLog LOG = XLog.getLog(SLAEmailEventListener.class);
078    private Set<SLAEvent.EventStatus> alertEvents;
079    public static String EMAIL_BODY_FIELD_SEPARATER = " - ";
080    public static String EMAIL_BODY_FIELD_INDENT = "  ";
081    public static String EMAIL_BODY_HEADER_SEPARATER = ":";
082
083    public enum EmailField {
084        EVENT_STATUS("SLA Status"), APP_TYPE("App Type"), APP_NAME("App Name"), USER("User"), JOBID("Job ID"), PARENT_JOBID(
085                "Parent Job ID"), JOB_URL("Job URL"), PARENT_JOB_URL("Parent Job URL"), NOMINAL_TIME("Nominal Time"),
086                EXPECTED_START_TIME("Expected Start Time"), ACTUAL_START_TIME("Actual Start Time"),
087                EXPECTED_END_TIME("Expected End Time"), ACTUAL_END_TIME("Actual End Time"),
088                EXPECTED_DURATION("Expected Duration (in mins)"),
089                ACTUAL_DURATION("Actual Duration (in mins)"), NOTIFICATION_MESSAGE("Notification Message"),
090                UPSTREAM_APPS("Upstream Apps"),
091                JOB_STATUS("Job Status");
092        private String name;
093
094        private EmailField(String name) {
095            this.name = name;
096        }
097
098        @Override
099        public String toString() {
100            return name;
101        }
102    };
103
104    @Override
105    public void init(Configuration conf) throws Exception {
106
107        oozieBaseUrl = ConfigurationService.get(conf, OOZIE_BASE_URL);
108        // Get SMTP properties from the configuration used in Email Action
109        String smtpHost = conf.get(EmailActionExecutor.EMAIL_SMTP_HOST, SMTP_HOST_DEFAULT);
110        String smtpPort = conf.get(EmailActionExecutor.EMAIL_SMTP_PORT, SMTP_PORT_DEFAULT);
111        Boolean smtpAuth = conf.getBoolean(EmailActionExecutor.EMAIL_SMTP_AUTH, SMTP_AUTH_DEFAULT);
112        String smtpUser = conf.get(EmailActionExecutor.EMAIL_SMTP_USER, "");
113        String smtpPassword = ConfigurationService.getPassword(EmailActionExecutor.EMAIL_SMTP_PASS, "");
114        String smtpConnectTimeout = conf.get(SMTP_CONNECTION_TIMEOUT, SMTP_CONNECTION_TIMEOUT_DEFAULT);
115        String smtpTimeout = conf.get(SMTP_TIMEOUT, SMTP_TIMEOUT_DEFAULT);
116
117        int blacklistTimeOut = Integer.valueOf(conf.get(BLACKLIST_CACHE_TIMEOUT, BLACKLIST_CACHE_TIMEOUT_DEFAULT));
118        blacklistFailCount = Integer.valueOf(conf.get(BLACKLIST_FAIL_COUNT, BLACKLIST_FAIL_COUNT_DEFAULT));
119
120        // blacklist email addresses causing SendFailedException with cache timeout
121        blackList = CacheBuilder.newBuilder()
122                .expireAfterWrite(blacklistTimeOut, TimeUnit.SECONDS)
123                .build(new CacheLoader<String, AtomicInteger>() {
124                    @Override
125                    public AtomicInteger load(String key) throws Exception {
126                        return new AtomicInteger();
127                    }
128                });
129
130        // Set SMTP properties
131        Properties properties = new Properties();
132        properties.setProperty("mail.smtp.host", smtpHost);
133        properties.setProperty("mail.smtp.port", smtpPort);
134        properties.setProperty("mail.smtp.auth", smtpAuth.toString());
135        properties.setProperty("mail.smtp.connectiontimeout", smtpConnectTimeout);
136        properties.setProperty("mail.smtp.timeout", smtpTimeout);
137
138        try {
139            fromAddr = new InternetAddress(conf.get("oozie.email.from.address", SMTP_SOURCE_DEFAULT));
140        }
141        catch (AddressException ae) {
142            LOG.error("Bad Source Address specified in oozie.email.from.address", ae);
143            throw ae;
144        }
145
146        if (!smtpAuth) {
147            session = Session.getInstance(properties);
148        }
149        else {
150            session = Session.getInstance(properties, new JavaMailAuthenticator(smtpUser, smtpPassword));
151        }
152
153        alertEvents = new HashSet<SLAEvent.EventStatus>();
154        String alertEventsStr = ConfigurationService.get(conf, SLAService.CONF_ALERT_EVENTS);
155        if (alertEventsStr != null) {
156            String[] alertEvt = alertEventsStr.split(",", -1);
157            for (String evt : alertEvt) {
158                alertEvents.add(SLAEvent.EventStatus.valueOf(evt));
159            }
160        }
161    }
162
163    @Override
164    public void destroy() {
165    }
166
167    private void sendSLAEmail(SLAEvent event) throws Exception {
168        // If no address is provided, the user did not want to send an email so simply log it and do nothing
169        if (event.getAlertContact() == null || event.getAlertContact().trim().length() == 0) {
170            LOG.info("No destination address provided; an SLA alert email will not be sent");
171        } else {
172            // Create and send an email
173            Message message = new MimeMessage(session);
174            setMessageHeader(message, event);
175            setMessageBody(message, event);
176            sendEmail(message);
177        }
178    }
179
180    @Override
181    public void onStartMiss(SLAEvent event) {
182        boolean flag = false;
183        if (event.getAlertEvents() == null) {
184            flag = alertEvents.contains(SLAEvent.EventStatus.START_MISS);
185        }
186        else if (event.getAlertEvents().contains(SLAEvent.EventStatus.START_MISS.name())) {
187            flag = true;
188        }
189
190        if (flag) {
191            try {
192                sendSLAEmail(event);
193            }
194            catch (Exception e) {
195                LOG.error("Failed to send StartMiss alert email", e);
196            }
197        }
198    }
199
200    @Override
201    public void onEndMiss(SLAEvent event) {
202        boolean flag = false;
203        if (event.getAlertEvents() == null) {
204            flag = alertEvents.contains(SLAEvent.EventStatus.END_MISS);
205        }
206        else if (event.getAlertEvents().contains(SLAEvent.EventStatus.END_MISS.name())) {
207            flag = true;
208        }
209
210        if (flag) {
211            try {
212                sendSLAEmail(event);
213            }
214            catch (Exception e) {
215                LOG.error("Failed to send EndMiss alert email", e);
216            }
217        }
218    }
219
220    @Override
221    public void onDurationMiss(SLAEvent event) {
222        boolean flag = false;
223        if (event.getAlertEvents() == null) {
224            flag = alertEvents.contains(SLAEvent.EventStatus.DURATION_MISS);
225        }
226        else if (event.getAlertEvents().contains(SLAEvent.EventStatus.DURATION_MISS.name())) {
227            flag = true;
228        }
229
230        if (flag) {
231            try {
232                sendSLAEmail(event);
233            }
234            catch (Exception e) {
235                LOG.error("Failed to send DurationMiss alert email", e);
236            }
237        }
238    }
239
240    private Address[] parseAddress(String str) {
241        Address[] addrs = null;
242        List<InternetAddress> addrList = new ArrayList<InternetAddress>();
243        String[] emails = str.split(ADDRESS_SEPARATOR, -1);
244
245        for (String email : emails) {
246            boolean isBlackListed = false;
247            AtomicInteger val = blackList.getIfPresent(email);
248            if(val != null){
249                isBlackListed = ( val.get() >= blacklistFailCount );
250            }
251            if (!isBlackListed) {
252                try {
253                    // turn on strict syntax check by setting 2nd argument true
254                    addrList.add(new InternetAddress(email, true));
255                }
256                catch (AddressException ae) {
257                    // simply skip bad address but do not throw exception
258                    LOG.error("Skipping bad destination address: " + email, ae);
259                }
260            }
261        }
262
263        if (addrList.size() > 0) {
264            addrs = (Address[]) addrList.toArray(new InternetAddress[addrList.size()]);
265        }
266
267        return addrs;
268    }
269
270    private void setMessageHeader(Message msg, SLAEvent event) throws MessagingException {
271        Address[] from = new InternetAddress[] { fromAddr };
272        Address[] to;
273        StringBuilder subject = new StringBuilder();
274
275        to = parseAddress(event.getAlertContact());
276        if (to == null) {
277            LOG.error("Destination address is null or invalid, stop sending SLA alert email");
278            throw new IllegalArgumentException("Destination address is not specified properly");
279        }
280        subject.append("OOZIE - SLA ");
281        subject.append(event.getEventStatus().name());
282        subject.append(" (AppName=");
283        subject.append(event.getAppName());
284        subject.append(", JobID=");
285        subject.append(event.getId());
286        subject.append(")");
287
288        try {
289            msg.addFrom(from);
290            msg.addRecipients(RecipientType.TO, to);
291            msg.setSubject(subject.toString());
292        }
293        catch (MessagingException me) {
294            LOG.error("Message Exception in setting message header of SLA alert email", me);
295            throw me;
296        }
297    }
298
299    private void setMessageBody(Message msg, SLAEvent event) throws MessagingException {
300        StringBuilder body = new StringBuilder();
301        printHeading(body, "Status");
302        printField(body, EmailField.EVENT_STATUS.toString(), event.getEventStatus());
303        printField(body, EmailField.JOB_STATUS.toString(), event.getJobStatus());
304        printField(body, EmailField.NOTIFICATION_MESSAGE.toString(), event.getNotificationMsg());
305
306        printHeading(body, "Job Details");
307        printField(body, EmailField.APP_NAME.toString(), event.getAppName());
308        printField(body, EmailField.APP_TYPE.toString(), event.getAppType());
309        printField(body, EmailField.USER.toString(), event.getUser());
310        printField(body, EmailField.JOBID.toString(), event.getId());
311        printField(body, EmailField.JOB_URL.toString(), getJobLink(event.getId()));
312        printField(body, EmailField.PARENT_JOBID.toString(), event.getParentId() != null ? event.getParentId() : "N/A");
313        printField(body, EmailField.PARENT_JOB_URL.toString(),
314                event.getParentId() != null ? getJobLink(event.getParentId()) : "N/A");
315        printField(body, EmailField.UPSTREAM_APPS.toString(), event.getUpstreamApps());
316
317        printHeading(body, "SLA Details");
318        printField(body, EmailField.NOMINAL_TIME.toString(), event.getNominalTime());
319        printField(body, EmailField.EXPECTED_START_TIME.toString(), event.getExpectedStart());
320        printField(body, EmailField.ACTUAL_START_TIME.toString(), event.getActualStart());
321        printField(body, EmailField.EXPECTED_END_TIME.toString(), event.getExpectedEnd());
322        printField(body, EmailField.ACTUAL_END_TIME.toString(), event.getActualEnd());
323        printField(body, EmailField.EXPECTED_DURATION.toString(), getDurationInMins(event.getExpectedDuration()));
324        printField(body, EmailField.ACTUAL_DURATION.toString(), getDurationInMins(event.getActualDuration()));
325
326        try {
327            msg.setText(body.toString());
328        }
329        catch (MessagingException me) {
330            LOG.error("Message Exception in setting message body of SLA alert email", me);
331            throw me;
332        }
333    }
334
335    private long getDurationInMins(long duration) {
336        if (duration < 0) {
337            return duration;
338        }
339        return duration / 60000; //Convert millis to minutes
340    }
341
342    private String getJobLink(String jobId) {
343        StringBuffer url = new StringBuffer();
344        String param = "/?job=";
345        url.append(oozieBaseUrl);
346        url.append(param);
347        url.append(jobId);
348        return url.toString();
349    }
350
351    private void printField(StringBuilder st, String name, Object value) {
352        String lineFeed = "\n";
353        if (value != null) {
354            st.append(EMAIL_BODY_FIELD_INDENT);
355            st.append(name);
356            st.append(EMAIL_BODY_FIELD_SEPARATER);
357            st.append(value);
358            st.append(lineFeed);
359        }
360    }
361
362    private void printHeading(StringBuilder st, String header) {
363        st.append(header);
364        st.append(EMAIL_BODY_HEADER_SEPARATER);
365        st.append("\n");
366    }
367
368    private void sendEmail(Message message) throws MessagingException {
369        try {
370            Transport.send(message);
371        }
372        catch (NoSuchProviderException se) {
373            LOG.error("Could not find an SMTP transport provider to email", se);
374            throw se;
375        }
376        catch (MessagingException me) {
377            LOG.error("Message Exception in transporting SLA alert email", me);
378            if (me instanceof SendFailedException) {
379                Address[] invalidAddrs = ((SendFailedException) me).getInvalidAddresses();
380                if (invalidAddrs != null && invalidAddrs.length > 0) {
381                    for (Address addr : invalidAddrs) {
382                        try {
383                            // 'get' method loads key into cache when it doesn't exist
384                            AtomicInteger val = blackList.get(addr.toString());
385                            val.incrementAndGet();
386                        }
387                        catch (Exception e) {
388                            LOG.debug("blacklist loading threw exception: " + e.getMessage());
389                        }
390                    }
391                }
392            }
393            throw me;
394        }
395    }
396
397    @VisibleForTesting
398    public void addBlackList(String email) throws Exception {
399        // this is for testing
400        if(email == null || email.equals("")){
401            return;
402        }
403        AtomicInteger val = blackList.get(email);
404        val.set(blacklistFailCount);
405    }
406
407    @Override
408    public void onStartMet(SLAEvent work) {
409    }
410
411    @Override
412    public void onEndMet(SLAEvent work) {
413    }
414
415    @Override
416    public void onDurationMet(SLAEvent work) {
417    }
418
419}