Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import java.io.IOException;
022import java.util.ArrayList;
023import java.util.Date;
024import java.util.HashMap;
025import java.util.List;
026import java.util.Locale;
027import java.util.Map;
028import java.util.regex.Matcher;
029import java.util.regex.Pattern;
030
031import org.apache.commons.lang.StringUtils;
032import org.apache.oozie.service.ConfigurationService;
033import org.apache.oozie.util.LogLine.MATCHED_PATTERN;
034
035import com.google.common.annotations.VisibleForTesting;
036
037/**
038 * Filter that will construct the regular expression that will be used to filter the log statement. And also checks if
039 * the given log message go through the filter. Filters that can be used are logLevel(Multi values separated by "|")
040 * jobId appName actionId token
041 */
042public class XLogFilter {
043
044    private static final int LOG_TIME_BUFFER = 2; // in min
045    public static String MAX_ACTIONLIST_SCAN_DURATION = "oozie.service.XLogStreamingService.actionlist.max.log.scan.duration";
046    public static String MAX_SCAN_DURATION = "oozie.service.XLogStreamingService.max.log.scan.duration";
047    private Map<String, Integer> logLevels;
048    private final Map<String, String> filterParams;
049    private static List<String> parameters = new ArrayList<String>();
050    private boolean noFilter;
051    private Pattern filterPattern;
052    private XLogUserFilterParam userLogFilter;
053    private Date endDate;
054    private Date startDate;
055    private boolean isActionList = false;
056    private String formattedEndDate;
057    private String formattedStartDate;
058    private String truncatedMessage;
059
060    // TODO Patterns to be read from config file
061    private static final String DEFAULT_REGEX = "[^\\]]*";
062
063    public static final String ALLOW_ALL_REGEX = "(.*)";
064    private static final String TIMESTAMP_REGEX = "(\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d,\\d\\d\\d)";
065    private static final String WHITE_SPACE_REGEX = "\\s+";
066    private static final String LOG_LEVEL_REGEX = "(\\w+)";
067    static final String PREFIX_REGEX = TIMESTAMP_REGEX + WHITE_SPACE_REGEX + LOG_LEVEL_REGEX
068            + WHITE_SPACE_REGEX;
069    private static final Pattern SPLITTER_PATTERN = Pattern.compile(PREFIX_REGEX + ALLOW_ALL_REGEX);
070
071    public XLogFilter() {
072        this(new XLogUserFilterParam());
073    }
074
075    public XLogFilter(XLogUserFilterParam userLogFilter) {
076        filterParams = new HashMap<String, String>();
077        for (int i = 0; i < parameters.size(); i++) {
078            filterParams.put(parameters.get(i), DEFAULT_REGEX);
079        }
080        logLevels = null;
081        noFilter = true;
082        filterPattern = null;
083        setUserLogFilter(userLogFilter);
084    }
085
086    public void setLogLevel(String logLevel) {
087        if (logLevel != null && logLevel.trim().length() > 0) {
088            this.logLevels = new HashMap<String, Integer>();
089            String[] levels = logLevel.split("\\|");
090            for (int i = 0; i < levels.length; i++) {
091                String s = levels[i].trim().toUpperCase();
092                try {
093                    XLog.Level.valueOf(s);
094                }
095                catch (Exception ex) {
096                    continue;
097                }
098                this.logLevels.put(levels[i].toUpperCase(), 1);
099            }
100        }
101    }
102
103    public void setParameter(String filterParam, String value) {
104        if (filterParams.containsKey(filterParam)) {
105            noFilter = false;
106            filterParams.put(filterParam, value);
107        }
108    }
109
110    public static void defineParameter(String filterParam) {
111        parameters.add(filterParam);
112    }
113
114    public boolean isFilterPresent() {
115        if (noFilter && logLevels == null) {
116            return false;
117        }
118        return true;
119    }
120
121    /**
122     * Checks if the logLevel and logMessage goes through the logFilter.
123     * @param logLine the log line
124     * @return true if line contains the permitted logLevel
125     */
126    public boolean splitsMatches(LogLine logLine) {
127        // Check whether logLine matched with filter
128        if (logLine.getMatchedPattern() != MATCHED_PATTERN.SPLIT) {
129            return false;
130        }
131        ArrayList<String> logParts = logLine.getLogParts();
132        if (getStartDate() != null) {
133            if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
134                return false;
135            }
136        }
137        String logLevel = logParts.get(1);
138        if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase(Locale.ENGLISH))) {
139            // line contains the permitted logLevel
140            return true;
141        }
142        else {
143            return false;
144        }
145    }
146
147    /**
148     * Checks if the logLevel and logMessage goes through the logFilter.
149     *
150     * @param logParts the arrayList of log parts
151     * @return true if the logLevel and logMessage goes through the logFilter
152     */
153    public boolean matches(ArrayList<String> logParts) {
154        if (getStartDate() != null) {
155            if (logParts.get(0).substring(0, 19).compareTo(getFormattedStartDate()) < 0) {
156                return false;
157            }
158        }
159        String logLevel = logParts.get(1);
160        String logMessage = logParts.get(2);
161        if (this.logLevels == null || this.logLevels.containsKey(logLevel.toUpperCase())) {
162            Matcher logMatcher = filterPattern.matcher(logMessage);
163            return logMatcher.matches();
164        }
165        else {
166            return false;
167        }
168    }
169
170    /**
171     * Splits the log line into timestamp, logLevel and remaining log message.
172     * Returns array containing timestamp, logLevel, and logMessage if the
173     * pattern matches i.e A new log statement, else returns null.
174     *
175     * @param logLine the line
176     * @return Array containing log level and log message
177     */
178    public ArrayList<String> splitLogMessage(String logLine) {
179        Matcher splitter = SPLITTER_PATTERN.matcher(logLine);
180        if (splitter.matches()) {
181            ArrayList<String> logParts = new ArrayList<String>();
182            logParts.add(splitter.group(1));// timestamp
183            logParts.add(splitter.group(2));// log level
184            logParts.add(splitter.group(3));// Log Message
185            return logParts;
186        }
187        else {
188            return null;
189        }
190    }
191
192    /**
193     * If <code>logLine</code> matches with <code>splitPattern</code>,
194     * <ol>
195     * <li>Split the log line into timestamp, logLevel and remaining log
196     * message.</li>
197     * <li>Record the parts of message in <code>logLine</code> to avoid regex
198     * matching in future.</li>
199     * <li>Record the pattern to which <code>logLine</code> has matched.</li>
200     * </ol>
201     * @param logLine the line to split
202     * @param splitPattern the pattern to use
203     */
204    public void splitLogMessage(LogLine logLine, Pattern splitPattern) {
205        Matcher splitterWithJobId = splitPattern.matcher(logLine.getLine());
206        Matcher allowAll = SPLITTER_PATTERN.matcher(logLine.getLine());
207        if (splitterWithJobId.matches()) {
208            ArrayList<String> logParts = new ArrayList<String>(3);
209            logParts.add(splitterWithJobId.group(1));// timestamp
210            logParts.add(splitterWithJobId.group(2));// log level
211            logParts.add(splitterWithJobId.group(3));// log message
212            logLine.setLogParts(logParts);
213            logLine.setMatchedPattern(MATCHED_PATTERN.SPLIT);
214        }
215        else if (allowAll.matches()) {
216            logLine.setMatchedPattern(MATCHED_PATTERN.GENENRIC);
217        }
218        else {
219            logLine.setMatchedPattern(MATCHED_PATTERN.NONE);
220        }
221    }
222
223    /**
224     * Constructs the regular expression according to the filter and assigns it
225     * to fileterPattarn. ".*" will be assigned if no filters are set.
226     */
227    public void constructPattern() {
228        if (noFilter && logLevels == null) {
229            filterPattern = Pattern.compile(ALLOW_ALL_REGEX);
230            return;
231        }
232        StringBuilder sb = new StringBuilder();
233        if (noFilter) {
234            sb.append("(.*)");
235        }
236        else {
237            sb.append("(.* ");
238            for (int i = 0; i < parameters.size(); i++) {
239                sb.append(parameters.get(i) + "\\[");
240                sb.append(filterParams.get(parameters.get(i)) + "\\] ");
241            }
242            sb.append(".*)");
243        }
244        if (!StringUtils.isEmpty(userLogFilter.getSearchText())) {
245            sb.append(userLogFilter.getSearchText() + ".*");
246        }
247        filterPattern = Pattern.compile(sb.toString());
248    }
249
250    public static void reset() {
251        parameters.clear();
252    }
253
254    public final Map<String, String> getFilterParams() {
255        return filterParams;
256    }
257
258    public XLogUserFilterParam getUserLogFilter() {
259        return userLogFilter;
260    }
261
262    public void setUserLogFilter(XLogUserFilterParam userLogFilter) {
263        this.userLogFilter = userLogFilter;
264        setLogLevel(userLogFilter.getLogLevel());
265    }
266
267    public Date getEndDate() {
268        return endDate;
269    }
270
271    public String getFormattedEndDate() {
272        return formattedEndDate;
273    }
274
275    public String getFormattedStartDate() {
276        return formattedStartDate;
277    }
278
279    public Date getStartDate() {
280        return startDate;
281    }
282
283    public boolean isDebugMode() {
284        return userLogFilter.isDebug();
285    }
286
287    public int getLogLimit() {
288        return userLogFilter.getLimit();
289    }
290
291    public String getDebugMessage() {
292        return new StringBuilder("Log start time = ").append(getStartDate()).append(". Log end time = ")
293                .append(getEndDate()).append(". User Log Filter = ").append(getUserLogFilter())
294                .append(System.getProperty("line.separator")).toString();
295    }
296
297    public boolean isActionList() {
298        return isActionList;
299    }
300
301    public void setActionList(boolean isActionList) {
302        this.isActionList = isActionList;
303    }
304
305    /**
306     * Calculate scan date
307     *
308     * @param jobStartTime the job start time
309     * @param jobEndTime the job end time
310     * @throws IOException Signals that an I/O exception has occurred.
311     */
312    public void calculateAndCheckDates(Date jobStartTime, Date jobEndTime) throws IOException {
313
314        // for testcase, otherwise jobStartTime and jobEndTime will be always
315        // set
316        if (jobStartTime == null || jobEndTime == null) {
317            return;
318        }
319
320        if (userLogFilter.getStartDate() != null) {
321            startDate = userLogFilter.getStartDate();
322        }
323        else if (userLogFilter.getStartOffset() != -1) {
324            startDate = adjustOffset(jobStartTime, userLogFilter.getStartOffset());
325        }
326        else {
327            startDate = new Date(jobStartTime.getTime());
328        }
329
330        if (userLogFilter.getEndDate() != null) {
331            endDate = userLogFilter.getEndDate();
332        }
333        else if (userLogFilter.getEndOffset() != -1) {
334            // If user has specified startdate as absolute then end offset will
335            // be on user start date,
336            // else end offset will be calculated on job startdate.
337            if (userLogFilter.getStartDate() != null) {
338                endDate = adjustOffset(startDate, userLogFilter.getEndOffset());
339            }
340            else {
341                endDate = adjustOffset(jobStartTime, userLogFilter.getEndOffset());
342            }
343        }
344        else {
345            endDate = new Date(jobEndTime.getTime());
346        }
347        // if recent offset is specified then start time = endtime - offset
348        if (getUserLogFilter().getRecent() != -1) {
349            startDate = adjustOffset(endDate, userLogFilter.getRecent() * -1);
350        }
351
352        // add buffer if dates are not absolute
353        if (userLogFilter.getStartDate() == null) {
354            startDate = adjustOffset(startDate, -LOG_TIME_BUFFER);
355        }
356        if (userLogFilter.getEndDate() == null) {
357            endDate = adjustOffset(endDate, LOG_TIME_BUFFER);
358        }
359
360        formattedEndDate = XLogUserFilterParam.dt.get().format(getEndDate());
361        formattedStartDate = XLogUserFilterParam.dt.get().format(getStartDate());
362
363        if (startDate.after(endDate)) {
364            throw new IOException(
365                    "Start time should be less than end time. startTime = " + startDate + " endTime = " + endDate);
366        }
367    }
368
369    /**
370     * validate date range.
371     *
372     * @param jobStartTime the job start time
373     * @param jobEndTime the job end time
374     * @throws IOException Signals that an I/O exception has occurred.
375     */
376    public void validateDateRange(Date jobStartTime, Date jobEndTime) throws IOException {
377        // for testcase, otherwise jobStartTime and jobEndTime will be always
378        // set
379        if (jobStartTime == null || jobEndTime == null) {
380            return;
381        }
382
383        long diffHours = (endDate.getTime() - startDate.getTime()) / (60 * 60 * 1000);
384        if (isActionList) {
385            int actionLogDuration = ConfigurationService.getInt(MAX_ACTIONLIST_SCAN_DURATION);
386            if (actionLogDuration == -1) {
387                return;
388            }
389            if (diffHours > actionLogDuration) {
390                setTruncatedMessage("Truncated logs to max log scan duration " + actionLogDuration + " hrs");
391                startDate = adjustOffset(endDate, -1 * actionLogDuration * 60);
392                startDate = adjustOffset(startDate, -1 * LOG_TIME_BUFFER);
393            }
394        }
395        else {
396            int logDuration = ConfigurationService.getInt(MAX_SCAN_DURATION);
397            if (logDuration == -1) {
398                return;
399            }
400            if (diffHours > logDuration) {
401                setTruncatedMessage("Truncated logs to max log scan duration " + logDuration + " hrs");
402                startDate = adjustOffset(endDate, -1 * logDuration * 60);
403                startDate = adjustOffset(startDate, -1 * LOG_TIME_BUFFER);
404            }
405        }
406    }
407
408    protected void setTruncatedMessage(String message) {
409        truncatedMessage = message;
410
411    }
412
413    public String getTruncatedMessage() {
414        if (StringUtils.isEmpty(truncatedMessage)) {
415            return truncatedMessage;
416        }
417        else {
418            return truncatedMessage + System.getProperty("line.separator");
419        }
420    }
421
422    /**
423     * Adjust offset, offset will always be in min.
424     *
425     * @param date the date
426     * @param offset the offset
427     * @return the date
428     * @throws IOException Signals that an I/O exception has occurred.
429     */
430    public Date adjustOffset(Date date, int offset) throws IOException {
431        return org.apache.commons.lang.time.DateUtils.addMinutes(date, offset);
432    }
433
434    public void setFilterPattern(Pattern filterPattern) {
435        this.filterPattern = filterPattern;
436    }
437
438    public Pattern getFilterPattern() {
439        return this.filterPattern;
440    }
441
442}