Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.coord.input.dependency;
020
021import java.io.DataInput;
022import java.io.DataOutput;
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.Collection;
026import java.util.Date;
027import java.util.HashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.Map.Entry;
031import java.util.Set;
032
033import org.apache.commons.lang.StringUtils;
034import org.apache.hadoop.io.Writable;
035import org.apache.oozie.CoordinatorActionBean;
036import org.apache.oozie.command.CommandException;
037import org.apache.oozie.command.coord.CoordCommandUtils;
038import org.apache.oozie.coord.CoordELFunctions;
039import org.apache.oozie.coord.input.logic.CoordInputLogicEvaluatorUtil;
040import org.apache.oozie.dependency.ActionDependency;
041import org.apache.oozie.util.DateUtils;
042import org.apache.oozie.util.WritableUtils;
043import org.jdom.Element;
044import org.jdom.JDOMException;
045
046public abstract class AbstractCoordInputDependency implements Writable, CoordInputDependency {
047    protected boolean isDependencyMet = false;
048    /*
049     * Transient variables only used for processing, not stored in DB.
050     */
051    protected transient Map<String, List<String>> missingDependenciesSet = new HashMap<String, List<String>>();
052    protected transient Map<String, List<String>> availableDependenciesSet = new HashMap<String, List<String>>();
053    protected Map<String, List<CoordInputInstance>> dependencyMap = new HashMap<String, List<CoordInputInstance>>();
054
055    public AbstractCoordInputDependency() {
056    }
057
058
059    public AbstractCoordInputDependency(Map<String, List<CoordInputInstance>> dependencyMap) {
060        this.dependencyMap = dependencyMap;
061        generateDependencies();
062    }
063
064    public void addInputInstanceList(String inputEventName, List<CoordInputInstance> inputInstanceList) {
065        dependencyMap.put(inputEventName, inputInstanceList);
066    }
067
068    public Map<String, List<CoordInputInstance>> getDependencyMap() {
069        return dependencyMap;
070    }
071
072    public void setDependencyMap(Map<String, List<CoordInputInstance>> dependencyMap) {
073        this.dependencyMap = dependencyMap;
074    }
075
076    public void addToAvailableDependencies(String dataSet, CoordInputInstance coordInputInstance) {
077        coordInputInstance.setAvailability(true);
078        List<String> availableSet = availableDependenciesSet.get(dataSet);
079        if (availableSet == null) {
080            availableSet = new ArrayList<String>();
081            availableDependenciesSet.put(dataSet, availableSet);
082        }
083        availableSet.add(coordInputInstance.getInputDataInstance());
084        removeFromMissingDependencies(dataSet, coordInputInstance);
085    }
086
087    public void removeFromMissingDependencies(String dataSet, CoordInputInstance coordInputInstance) {
088        coordInputInstance.setAvailability(true);
089        List<String> missingSet = missingDependenciesSet.get(dataSet);
090        if (missingSet != null) {
091            missingSet.remove(coordInputInstance.getInputDataInstance());
092            if (missingSet.isEmpty()) {
093                missingDependenciesSet.remove(dataSet);
094            }
095        }
096
097    }
098
099    public void addToMissingDependencies(String dataSet, CoordInputInstance coordInputInstance) {
100        List<String> availableSet = missingDependenciesSet.get(dataSet);
101        if (availableSet == null) {
102            availableSet = new ArrayList<String>();
103        }
104        availableSet.add(coordInputInstance.getInputDataInstance());
105        missingDependenciesSet.put(dataSet, availableSet);
106
107    }
108
109    protected void generateDependencies() {
110        try {
111            missingDependenciesSet = new HashMap<String, List<String>>();
112            availableDependenciesSet = new HashMap<String, List<String>>();
113
114            for (Entry<String, List<CoordInputInstance>> entry : dependencyMap.entrySet()) {
115                String key = entry.getKey();
116                for (CoordInputInstance coordInputInstance : entry.getValue()) {
117                    if (coordInputInstance.isAvailable()) {
118                        addToAvailableDependencies(key, coordInputInstance);
119                    }
120                    else {
121                        addToMissingDependencies(key, coordInputInstance);
122                    }
123                }
124            }
125        }
126        catch (Exception e) {
127            throw new RuntimeException(e);
128        }
129
130    }
131
132    public List<String> getAvailableDependencies(String dataSet) {
133        if (availableDependenciesSet.get(dataSet) != null) {
134            return availableDependenciesSet.get(dataSet);
135        }
136        else {
137            return new ArrayList<String>();
138        }
139
140    }
141
142    public String getMissingDependencies(String dataSet) {
143        StringBuilder sb = new StringBuilder();
144        for (String dependencies : missingDependenciesSet.get(dataSet)) {
145            sb.append(dependencies).append("#");
146        }
147        return sb.toString();
148    }
149
150    public void addToAvailableDependencies(String dataSet, String availableSet) {
151        List<CoordInputInstance> list = dependencyMap.get(dataSet);
152        if (list == null) {
153            list = new ArrayList<CoordInputInstance>();
154            dependencyMap.put(dataSet, list);
155        }
156
157        for (String available : availableSet.split(CoordELFunctions.INSTANCE_SEPARATOR)) {
158            CoordInputInstance coordInstance = new CoordInputInstance(available, true);
159            list.add(coordInstance);
160            addToAvailableDependencies(dataSet, coordInstance);
161        }
162
163    }
164
165    public String getMissingDependencies() {
166        StringBuilder sb = new StringBuilder();
167        if (missingDependenciesSet != null) {
168            for (List<String> dependenciesList : missingDependenciesSet.values()) {
169                for (String dependencies : dependenciesList) {
170                    sb.append(dependencies).append("#");
171                }
172            }
173        }
174        return sb.toString();
175    }
176
177    public List<String> getMissingDependenciesAsList() {
178        List<String> missingDependencies = new ArrayList<String>();
179        for (List<String> dependenciesList : missingDependenciesSet.values()) {
180            missingDependencies.addAll(dependenciesList);
181        }
182        return missingDependencies;
183    }
184
185    public List<String> getAvailableDependenciesAsList() {
186        List<String> availableDependencies = new ArrayList<String>();
187        for (List<String> dependenciesList : availableDependenciesSet.values()) {
188            availableDependencies.addAll(dependenciesList);
189
190        }
191        return availableDependencies;
192    }
193
194    public String serialize() throws IOException {
195        return CoordInputDependencyFactory.getMagicNumber()
196                + new String(WritableUtils.toByteArray(this), CoordInputDependencyFactory.CHAR_ENCODING);
197
198    }
199
200    public String getListAsString(List<String> dataSets) {
201        StringBuilder sb = new StringBuilder();
202        for (String dependencies : dataSets) {
203            sb.append(dependencies).append("#");
204        }
205
206        return sb.toString();
207    }
208
209    public void setDependencyMet(boolean isDependencyMeet) {
210        this.isDependencyMet = isDependencyMeet;
211    }
212
213    public boolean isDependencyMet() {
214        return missingDependenciesSet.isEmpty() || isDependencyMet;
215    }
216
217    public boolean isUnResolvedDependencyMet() {
218        return false;
219    }
220
221
222    @Override
223    public void addToAvailableDependencies(Collection<String> availableList) {
224        for (Entry<String, List<CoordInputInstance>> dependenciesList : dependencyMap.entrySet()) {
225            for (CoordInputInstance coordInputInstance : dependenciesList.getValue()) {
226                if (availableList.contains(coordInputInstance.getInputDataInstance()))
227                    addToAvailableDependencies(dependenciesList.getKey(), coordInputInstance);
228            }
229        }
230    }
231
232    @Override
233    public ActionDependency checkPushMissingDependencies(CoordinatorActionBean coordAction,
234            boolean registerForNotification) throws CommandException, IOException,
235            JDOMException {
236        boolean status = new CoordInputLogicEvaluatorUtil(coordAction).checkPushDependencies();
237        if (status) {
238            coordAction.getPushInputDependencies().setDependencyMet(true);
239        }
240        return new ActionDependency(coordAction.getPushInputDependencies().getMissingDependenciesAsList(), coordAction
241                .getPushInputDependencies().getAvailableDependenciesAsList());
242
243    }
244
245    public boolean checkPullMissingDependencies(CoordinatorActionBean coordAction,
246            StringBuilder existList, StringBuilder nonExistList) throws IOException, JDOMException {
247        boolean status = new CoordInputLogicEvaluatorUtil(coordAction).checkPullMissingDependencies();
248        if (status) {
249            coordAction.getPullInputDependencies().setDependencyMet(true);
250        }
251        return status;
252
253    }
254
255    public boolean isChangeInDependency(StringBuilder nonExistList, String missingDependencies,
256            StringBuilder nonResolvedList, boolean status) {
257        if (!StringUtils.isEmpty(missingDependencies)) {
258            return !missingDependencies.equals(getMissingDependencies());
259        }
260        else {
261            return true;
262        }
263    }
264
265    @SuppressWarnings("unchecked")
266    public boolean checkUnresolved(CoordinatorActionBean coordAction, Element eAction)
267            throws Exception {
268        String actualTimeStr = eAction.getAttributeValue("action-actual-time");
269        Element inputList = eAction.getChild("input-events", eAction.getNamespace());
270        Date actualTime = null;
271        if (actualTimeStr == null) {
272            actualTime = new Date();
273        }
274        else {
275            actualTime = DateUtils.parseDateOozieTZ(actualTimeStr);
276        }
277        if (inputList == null) {
278            return true;
279        }
280        List<Element> eDataEvents = inputList.getChildren("data-in", eAction.getNamespace());
281        for (Element dEvent : eDataEvents) {
282            if (dEvent.getChild(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, dEvent.getNamespace()) == null) {
283                continue;
284            }
285            String unResolvedInstance = dEvent.getChild(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG,
286                    dEvent.getNamespace()).getTextTrim();
287            String name = dEvent.getAttribute("name").getValue();
288            addUnResolvedList(name, unResolvedInstance);
289        }
290        return new CoordInputLogicEvaluatorUtil(coordAction).checkUnResolved(actualTime);
291    }
292
293    @Override
294    public void write(DataOutput out) throws IOException {
295        WritableUtils.writeStringAsBytes(out,INTERNAL_VERSION_ID);
296        out.writeBoolean(isDependencyMet);
297        WritableUtils.writeMapWithList(out, dependencyMap);
298    }
299
300    @Override
301    public void readFields(DataInput in) throws IOException {
302        WritableUtils.readBytesAsString(in);
303        this.isDependencyMet = in.readBoolean();
304        dependencyMap = WritableUtils.readMapWithList(in, CoordInputInstance.class);
305        generateDependencies();
306    }
307
308    public boolean isDataSetResolved(String dataSet){
309        if(getAvailableDependencies(dataSet) ==null|| getDependencyMap().get(dataSet) == null){
310            return false;
311        }
312        return getAvailableDependencies(dataSet).size() == getDependencyMap().get(dataSet).size();
313    }
314
315    @Override
316    public Map<String, ActionDependency> getMissingDependencies(CoordinatorActionBean coordAction)
317            throws CommandException, IOException, JDOMException {
318        Map<String, ActionDependency> missingDependenciesMap = new HashMap<String, ActionDependency>();
319        for (String key : missingDependenciesSet.keySet()) {
320            missingDependenciesMap.put(key, new ActionDependency(missingDependenciesSet.get(key), new ArrayList<String>()));
321        }
322        return missingDependenciesMap;
323    }
324
325    public String getFirstMissingDependency() {
326        return null;
327    }
328
329}