001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.util;
020
021import java.io.File;
022import java.text.ParseException;
023import java.text.SimpleDateFormat;
024import java.util.ArrayList;
025import java.util.Collections;
026import java.util.List;
027import java.util.concurrent.Semaphore;
028
029import org.apache.log4j.Appender;
030import org.apache.log4j.pattern.ExtrasPatternParser;
031import org.apache.log4j.rolling.RollingPolicyBase;
032import org.apache.log4j.rolling.RolloverDescription;
033import org.apache.log4j.rolling.TimeBasedRollingPolicy;
034import org.apache.log4j.rolling.TriggeringPolicy;
035import org.apache.log4j.spi.LoggingEvent;
036import org.apache.log4j.pattern.LiteralPatternConverter;
037import org.apache.oozie.service.Services;
038import org.apache.oozie.service.XLogService;
039
040/**
041 * Has the same behavior as the TimeBasedRollingPolicy.  Additionally, it will delete older logs (MaxHistory determines how many
042 * older logs are retained).
043 */
044public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy {
045
046    /**
047     * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it
048     */
049    private TimeBasedRollingPolicy tbrp;
050
051    private Semaphore deleteSem;
052
053    private Thread deleteThread;
054
055    private int maxHistory = 720;       // (720 hours / 24 hours per day = 30 days) as default
056
057    String oozieLogDir;
058    String logFileName;
059
060    public int getMaxHistory() {
061        return maxHistory;
062    }
063
064    public void setMaxHistory(int maxHistory) {
065        this.maxHistory = maxHistory;
066    }
067
068    public OozieRollingPolicy() {
069        deleteSem = new Semaphore(1);
070        deleteThread = new Thread();
071        tbrp = new TimeBasedRollingPolicy();
072    }
073
074
075    @SuppressWarnings("rawtypes")
076    public void setFileNamePattern(String fnp) {
077        super.setFileNamePattern(fnp);
078        List converters = new ArrayList();
079        StringBuffer file = new StringBuffer();
080        ExtrasPatternParser
081                .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules());
082        if (!converters.isEmpty()) {
083            ((LiteralPatternConverter) converters.get(0)).format(null, file);
084            File f = new File(file.toString());
085            oozieLogDir = f.getParent();
086            logFileName = f.getName();
087        }
088        else {
089            XLogService xls = Services.get().get(XLogService.class);
090            oozieLogDir = xls.getOozieLogPath();
091            logFileName = xls.getOozieLogName();
092        }
093
094    }
095    @Override
096    public void activateOptions() {
097        super.activateOptions();
098        tbrp.setFileNamePattern(getFileNamePattern());
099        tbrp.activateOptions();
100    }
101
102    @Override
103    public RolloverDescription initialize(String file, boolean append) throws SecurityException {
104        return tbrp.initialize(file, append);
105    }
106
107    @Override
108    public RolloverDescription rollover(final String activeFile) throws SecurityException {
109        return tbrp.rollover(activeFile);
110    }
111
112    @Override
113    public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename,
114    final long fileLength) {
115        if (maxHistory >= 0) {  // -1 = disable
116            // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting
117            // up to delete
118            // the old logs
119            if (deleteSem.tryAcquire()) {
120                if (!deleteThread.isAlive()) {
121                    // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else
122                    deleteThread = new Thread() {
123                        @Override
124                        public void run() {
125                            deleteOldFiles();
126                        }
127                    };
128                    deleteThread.start();
129                }
130                deleteSem.release();
131            }
132        }
133        return tbrp.isTriggeringEvent(appender, event, filename, fileLength);
134    }
135
136    private void deleteOldFiles() {
137        ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
138        if (oozieLogDir != null && logFileName != null) {
139            String[] children = new File(oozieLogDir).list();
140            if (children != null) {
141                for (String child : children) {
142                    if (child.startsWith(logFileName) && !child.equals(logFileName)) {
143                        File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child);
144                        if (child.endsWith(".gz")) {
145                            long gzFileCreationTime = getGZFileCreationTime(child);
146                            if (gzFileCreationTime != -1) {
147                                fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime));
148                            }
149                        }
150                        else {
151                            long modTime = childFile.lastModified();
152                            fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime));
153                        }
154                    }
155                }
156            }
157        }
158
159        if (fileList.size() > maxHistory) {
160            Collections.sort(fileList);
161
162            for (int i = maxHistory; i < fileList.size(); i++) {
163                new File(fileList.get(i).getFileName()).delete();
164            }
165        }
166    }
167
168    private long getGZFileCreationTime(String fileName) {
169        SimpleDateFormat formatter;
170        String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz
171        if (date.length() == 10) {
172            formatter = new SimpleDateFormat("yyyy-MM-dd");
173        }
174        else {
175            formatter = new SimpleDateFormat("yyyy-MM-dd-HH");
176        }
177        try {
178            return formatter.parse(date).getTime();
179        }
180        catch (ParseException e) {
181            return -1;
182        }
183    }
184
185    class FileInfo implements Comparable<FileInfo> {
186        String fileName;
187        long modTime;
188
189        public FileInfo(String fileName, long modTime) {
190            this.fileName = fileName;
191            this.modTime = modTime;
192        }
193
194        public String getFileName() {
195            return fileName;
196        }
197
198        public long getModTime() {
199            return modTime;
200        }
201
202        public int compareTo(FileInfo fileInfo) {
203            // Note: the order is the reverse of XLogStreamer.FileInfo
204            long diff = fileInfo.modTime - this.modTime;
205            if (diff > 0) {
206                return 1;
207            }
208            else if (diff < 0) {
209                return -1;
210            }
211            else {
212                return 0;
213            }
214        }
215    }
216
217}