001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.oozie.util; 020 021import java.io.File; 022import java.text.ParseException; 023import java.text.SimpleDateFormat; 024import java.util.ArrayList; 025import java.util.Collections; 026import java.util.List; 027import java.util.concurrent.Semaphore; 028 029import org.apache.log4j.Appender; 030import org.apache.log4j.pattern.ExtrasPatternParser; 031import org.apache.log4j.rolling.RollingPolicyBase; 032import org.apache.log4j.rolling.RolloverDescription; 033import org.apache.log4j.rolling.TimeBasedRollingPolicy; 034import org.apache.log4j.rolling.TriggeringPolicy; 035import org.apache.log4j.spi.LoggingEvent; 036import org.apache.log4j.pattern.LiteralPatternConverter; 037import org.apache.oozie.service.Services; 038import org.apache.oozie.service.XLogService; 039 040/** 041 * Has the same behavior as the TimeBasedRollingPolicy. Additionally, it will delete older logs (MaxHistory determines how many 042 * older logs are retained). 043 */ 044public class OozieRollingPolicy extends RollingPolicyBase implements TriggeringPolicy { 045 046 /** 047 * Unfortunately, TimeBasedRollingPolicy is declared final, so we can't subclass it; instead, we have to wrap it 048 */ 049 private TimeBasedRollingPolicy tbrp; 050 051 private Semaphore deleteSem; 052 053 private Thread deleteThread; 054 055 private int maxHistory = 720; // (720 hours / 24 hours per day = 30 days) as default 056 057 String oozieLogDir; 058 String logFileName; 059 060 public int getMaxHistory() { 061 return maxHistory; 062 } 063 064 public void setMaxHistory(int maxHistory) { 065 this.maxHistory = maxHistory; 066 } 067 068 public OozieRollingPolicy() { 069 deleteSem = new Semaphore(1); 070 deleteThread = new Thread(); 071 tbrp = new TimeBasedRollingPolicy(); 072 } 073 074 075 @SuppressWarnings("rawtypes") 076 public void setFileNamePattern(String fnp) { 077 super.setFileNamePattern(fnp); 078 List converters = new ArrayList(); 079 StringBuffer file = new StringBuffer(); 080 ExtrasPatternParser 081 .parse(fnp, converters, new ArrayList(), null, ExtrasPatternParser.getFileNamePatternRules()); 082 if (!converters.isEmpty()) { 083 ((LiteralPatternConverter) converters.get(0)).format(null, file); 084 File f = new File(file.toString()); 085 oozieLogDir = f.getParent(); 086 logFileName = f.getName(); 087 } 088 else { 089 XLogService xls = Services.get().get(XLogService.class); 090 oozieLogDir = xls.getOozieLogPath(); 091 logFileName = xls.getOozieLogName(); 092 } 093 094 } 095 @Override 096 public void activateOptions() { 097 super.activateOptions(); 098 tbrp.setFileNamePattern(getFileNamePattern()); 099 tbrp.activateOptions(); 100 } 101 102 @Override 103 public RolloverDescription initialize(String file, boolean append) throws SecurityException { 104 return tbrp.initialize(file, append); 105 } 106 107 @Override 108 public RolloverDescription rollover(final String activeFile) throws SecurityException { 109 return tbrp.rollover(activeFile); 110 } 111 112 @Override 113 public boolean isTriggeringEvent(final Appender appender, final LoggingEvent event, final String filename, 114 final long fileLength) { 115 if (maxHistory >= 0) { // -1 = disable 116 // Only delete old logs if we're not already deleting logs and another thread hasn't already started setting 117 // up to delete 118 // the old logs 119 if (deleteSem.tryAcquire()) { 120 if (!deleteThread.isAlive()) { 121 // Do the actual deleting in a new thread in case its slow so we don't bottleneck anything else 122 deleteThread = new Thread() { 123 @Override 124 public void run() { 125 deleteOldFiles(); 126 } 127 }; 128 deleteThread.start(); 129 } 130 deleteSem.release(); 131 } 132 } 133 return tbrp.isTriggeringEvent(appender, event, filename, fileLength); 134 } 135 136 private void deleteOldFiles() { 137 ArrayList<FileInfo> fileList = new ArrayList<FileInfo>(); 138 if (oozieLogDir != null && logFileName != null) { 139 String[] children = new File(oozieLogDir).list(); 140 if (children != null) { 141 for (String child : children) { 142 if (child.startsWith(logFileName) && !child.equals(logFileName)) { 143 File childFile = new File(new File(oozieLogDir).getAbsolutePath(), child); 144 if (child.endsWith(".gz")) { 145 long gzFileCreationTime = getGZFileCreationTime(child); 146 if (gzFileCreationTime != -1) { 147 fileList.add(new FileInfo(childFile.getAbsolutePath(), gzFileCreationTime)); 148 } 149 } 150 else { 151 long modTime = childFile.lastModified(); 152 fileList.add(new FileInfo(childFile.getAbsolutePath(), modTime)); 153 } 154 } 155 } 156 } 157 } 158 159 if (fileList.size() > maxHistory) { 160 Collections.sort(fileList); 161 162 for (int i = maxHistory; i < fileList.size(); i++) { 163 new File(fileList.get(i).getFileName()).delete(); 164 } 165 } 166 } 167 168 private long getGZFileCreationTime(String fileName) { 169 SimpleDateFormat formatter; 170 String date = fileName.substring(logFileName.length(), fileName.length() - 3); //3 for .gz 171 if (date.length() == 10) { 172 formatter = new SimpleDateFormat("yyyy-MM-dd"); 173 } 174 else { 175 formatter = new SimpleDateFormat("yyyy-MM-dd-HH"); 176 } 177 try { 178 return formatter.parse(date).getTime(); 179 } 180 catch (ParseException e) { 181 return -1; 182 } 183 } 184 185 class FileInfo implements Comparable<FileInfo> { 186 String fileName; 187 long modTime; 188 189 public FileInfo(String fileName, long modTime) { 190 this.fileName = fileName; 191 this.modTime = modTime; 192 } 193 194 public String getFileName() { 195 return fileName; 196 } 197 198 public long getModTime() { 199 return modTime; 200 } 201 202 public int compareTo(FileInfo fileInfo) { 203 // Note: the order is the reverse of XLogStreamer.FileInfo 204 long diff = fileInfo.modTime - this.modTime; 205 if (diff > 0) { 206 return 1; 207 } 208 else if (diff < 0) { 209 return -1; 210 } 211 else { 212 return 0; 213 } 214 } 215 } 216 217}