Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.oozie.dependency.hcat;
020
021import java.io.IOException;
022import java.net.InetAddress;
023import java.net.URI;
024import java.net.URISyntaxException;
025import java.util.ArrayList;
026import java.util.Collection;
027import java.util.Collections;
028import java.util.HashMap;
029import java.util.HashSet;
030import java.util.Iterator;
031import java.util.List;
032import java.util.Map;
033import java.util.Map.Entry;
034import java.util.Set;
035import java.util.concurrent.ConcurrentHashMap;
036import java.util.concurrent.ConcurrentMap;
037import org.apache.hadoop.conf.Configuration;
038import org.apache.oozie.service.ConfigurationService;
039import org.apache.oozie.service.HCatAccessorService;
040import org.apache.oozie.service.Services;
041import org.apache.oozie.util.HCatURI;
042import org.apache.oozie.util.XLog;
043
044public class SimpleHCatDependencyCache implements HCatDependencyCache {
045
046    private static XLog LOG = XLog.getLog(SimpleHCatDependencyCache.class);
047    private static String DELIMITER = ";";
048
049    public static final String USE_CANONICAL_HOSTNAME ="oozie.service.HCatAccessorService.jms.use.canonical.hostname";
050    private boolean useCanonicalHostName = false;
051
052
053
054    /**
055     * Map of server;db;table - sorter partition key order (country;dt;state) - sorted partition
056     * value (us;20120101;CA) - Collection of waiting actions (actionID and original hcat uri as
057     * string).
058     */
059    private ConcurrentMap<String, ConcurrentMap<String, Map<String, Collection<WaitingAction>>>> missingDeps;
060
061    /**
062     * Map of actionIDs and collection of available URIs
063     */
064    private ConcurrentMap<String, Collection<String>> availableDeps;
065
066    /**
067     * Map of actionIDs and partitions for reverse-lookup in purging
068     */
069    private ConcurrentMap<String, ConcurrentMap<String, Collection<String>>> actionPartitionMap;
070
071    @Override
072    public void init(Configuration conf) {
073        missingDeps = new ConcurrentHashMap<String, ConcurrentMap<String, Map<String, Collection<WaitingAction>>>>();
074        availableDeps = new ConcurrentHashMap<String, Collection<String>>();
075        actionPartitionMap = new ConcurrentHashMap<String, ConcurrentMap<String, Collection<String>>>();
076        useCanonicalHostName = ConfigurationService.getBoolean(USE_CANONICAL_HOSTNAME);
077    }
078
079    @Override
080    public void addMissingDependency(HCatURI hcatURI, String actionID) {
081        String tableKey = canonicalizeHostname(hcatURI.getServer()) + DELIMITER + hcatURI.getDb() + DELIMITER
082                + hcatURI.getTable();
083        SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
084        // Partition keys seperated by ;. For eg: date;country;state
085        String partKey = sortedPKV.getPartKeys();
086        // Partition values seperated by ;. For eg: 20120101;US;CA
087        String partVal = sortedPKV.getPartVals();
088        ConcurrentMap<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
089        if (partKeyPatterns == null) {
090            partKeyPatterns = new ConcurrentHashMap<String, Map<String, Collection<WaitingAction>>>();
091            ConcurrentMap<String, Map<String, Collection<WaitingAction>>> existingMap = missingDeps.putIfAbsent(
092                    tableKey, partKeyPatterns);
093            if (existingMap != null) {
094                partKeyPatterns = existingMap;
095            }
096        }
097        ConcurrentMap<String, Collection<String>> partitionMap = actionPartitionMap.get(actionID);
098        if (partitionMap == null) {
099            partitionMap = new ConcurrentHashMap<String, Collection<String>>();
100            ConcurrentMap<String, Collection<String>> existingPartMap = actionPartitionMap.putIfAbsent(actionID,
101                    partitionMap);
102            if (existingPartMap != null) {
103                partitionMap = existingPartMap;
104            }
105        }
106        synchronized (partitionMap) {
107            Collection<String> partKeys = partitionMap.get(tableKey);
108            if (partKeys == null) {
109                partKeys = new ArrayList<String>();
110            }
111            partKeys.add(partKey);
112            partitionMap.put(tableKey, partKeys);
113        }
114        synchronized (partKeyPatterns) {
115            missingDeps.put(tableKey, partKeyPatterns); // To handle race condition with removal of partKeyPatterns
116            Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
117            if (partValues == null) {
118                partValues = new HashMap<String, Collection<WaitingAction>>();
119                partKeyPatterns.put(partKey, partValues);
120            }
121            Collection<WaitingAction> waitingActions = partValues.get(partVal);
122            if (waitingActions == null) {
123                waitingActions = new HashSet<WaitingAction>();
124                partValues.put(partVal, waitingActions);
125            }
126            waitingActions.add(new WaitingAction(actionID, hcatURI.toURIString()));
127        }
128    }
129
130    @Override
131    public boolean removeMissingDependency(HCatURI hcatURI, String actionID) {
132        String tableKey = canonicalizeHostname(hcatURI.getServer()) + DELIMITER + hcatURI.getDb() + DELIMITER
133                + hcatURI.getTable();
134        SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
135        String partKey = sortedPKV.getPartKeys();
136        String partVal = sortedPKV.getPartVals();
137        Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
138        if (partKeyPatterns == null) {
139            LOG.warn("Remove missing dependency - Missing table entry - uri={0}, actionID={1}",
140                    hcatURI.toURIString(), actionID);
141            return false;
142        }
143        ConcurrentMap<String, Collection<String>> partitionMap = actionPartitionMap.get(actionID);
144        if (partitionMap != null) {
145            synchronized (partitionMap) {
146                Collection<String> partKeys = partitionMap.get(tableKey);
147                if (partKeys != null) {
148                    partKeys.remove(partKey);
149                }
150                if (partKeys.size() == 0) {
151                    partitionMap.remove(tableKey);
152                }
153                if (partitionMap.size() == 0) {
154                    actionPartitionMap.remove(actionID);
155                }
156            }
157        }
158
159        synchronized(partKeyPatterns) {
160            Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
161            if (partValues == null) {
162                LOG.warn("Remove missing dependency - Missing partition pattern - uri={0}, actionID={1}",
163                        hcatURI.toURIString(), actionID);
164                return false;
165            }
166            Collection<WaitingAction> waitingActions = partValues.get(partVal);
167            if (waitingActions == null) {
168                LOG.warn("Remove missing dependency - Missing partition value - uri={0}, actionID={1}",
169                        hcatURI.toURIString(), actionID);
170                return false;
171            }
172            boolean removed = waitingActions.remove(new WaitingAction(actionID, hcatURI.toURIString()));
173            if (!removed) {
174                LOG.warn("Remove missing dependency - Missing action ID - uri={0}, actionID={1}",
175                        hcatURI.toURIString(), actionID);
176            }
177            if (waitingActions.isEmpty()) {
178                partValues.remove(partVal);
179                if (partValues.isEmpty()) {
180                    partKeyPatterns.remove(partKey);
181                }
182                if (partKeyPatterns.isEmpty()) {
183                    missingDeps.remove(tableKey);
184                    // Close JMS session. Stop listening on topic
185                    HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
186                    hcatService.unregisterFromNotification(hcatURI);
187                }
188            }
189            return removed;
190        }
191    }
192
193    @Override
194    public Collection<String> getWaitingActions(HCatURI hcatURI) {
195        String tableKey = canonicalizeHostname(hcatURI.getServer()) + DELIMITER + hcatURI.getDb() + DELIMITER
196                + hcatURI.getTable();
197        SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
198        String partKey = sortedPKV.getPartKeys();
199        String partVal = sortedPKV.getPartVals();
200        Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
201        if (partKeyPatterns == null) {
202            return null;
203        }
204        Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
205        if (partValues == null) {
206            return null;
207        }
208        Collection<WaitingAction> waitingActions = partValues.get(partVal);
209        if (waitingActions == null) {
210            return null;
211        }
212        Collection<String> actionIDs = new ArrayList<String>();
213        URI uri = hcatURI.getURI();
214        String uriString = null;
215        try {
216            uriString = new URI(uri.getScheme(), canonicalizeHostname(uri.getAuthority()), uri.getPath(),
217                    uri.getQuery(), uri.getFragment()).toString();
218        }
219        catch (URISyntaxException e) {
220            uriString = hcatURI.toURIString();
221        }
222
223        for (WaitingAction action : waitingActions) {
224            if (action.getDependencyURI().equals(uriString)) {
225                actionIDs.add(action.getActionID());
226            }
227        }
228        return actionIDs;
229    }
230
231    @Override
232    public Collection<String> markDependencyAvailable(String server, String db, String table,
233            Map<String, String> partitions) {
234        String tableKey = canonicalizeHostname(server) + DELIMITER + db + DELIMITER + table;
235        Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
236        if (partKeyPatterns == null) {
237            LOG.warn("Got partition available notification for " + tableKey
238                    + ". Unexpected and should not be listening to topic. Unregistering topic");
239            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
240            hcatService.unregisterFromNotification(server, db, table);
241            return null;
242        }
243        Collection<String> actionsWithAvailDep = new HashSet<String>();
244        List<String> partKeysToRemove = new ArrayList<String>();
245        StringBuilder partValSB = new StringBuilder();
246        synchronized (partKeyPatterns) {
247            // If partition patterns are date, date;country and date;country;state,
248            // construct the partition values for each pattern from the available partitions map and
249            // for the matching value in the dependency map, get the waiting actions.
250            for (Entry<String, Map<String, Collection<WaitingAction>>> entry : partKeyPatterns.entrySet()) {
251                String[] partKeys = entry.getKey().split(DELIMITER);
252                partValSB.setLength(0);
253                for (String key : partKeys) {
254                    partValSB.append(partitions.get(key)).append(DELIMITER);
255                }
256                partValSB.setLength(partValSB.length() - 1);
257
258                Map<String, Collection<WaitingAction>> partValues = entry.getValue();
259                String partVal = partValSB.toString();
260                Collection<WaitingAction> wActions = entry.getValue().get(partVal);
261                if (wActions == null)
262                    continue;
263                for (WaitingAction wAction : wActions) {
264                    String actionID = wAction.getActionID();
265                    actionsWithAvailDep.add(actionID);
266                    Collection<String> depURIs = availableDeps.get(actionID);
267                    if (depURIs == null) {
268                        depURIs = new ArrayList<String>();
269                        Collection<String> existing = availableDeps.putIfAbsent(actionID, depURIs);
270                        if (existing != null) {
271                            depURIs = existing;
272                        }
273                    }
274                    synchronized (depURIs) {
275                        depURIs.add(wAction.getDependencyURI());
276                        availableDeps.put(actionID, depURIs);
277                    }
278                }
279                partValues.remove(partVal);
280                if (partValues.isEmpty()) {
281                    partKeysToRemove.add(entry.getKey());
282                }
283            }
284            for (String partKey : partKeysToRemove) {
285                partKeyPatterns.remove(partKey);
286            }
287            if (partKeyPatterns.isEmpty()) {
288                missingDeps.remove(tableKey);
289                // Close JMS session. Stop listening on topic
290                HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
291                hcatService.unregisterFromNotification(server, db, table);
292            }
293        }
294        return actionsWithAvailDep;
295    }
296
297    @Override
298    public Collection<String> getAvailableDependencyURIs(String actionID) {
299        Collection<String> available = availableDeps.get(actionID);
300        if (available !=  null) {
301            // Return a copy
302            available = new ArrayList<String>(available);
303        }
304        return available;
305    }
306
307    @Override
308    public boolean removeAvailableDependencyURIs(String actionID, Collection<String> dependencyURIs) {
309        if (!availableDeps.containsKey(actionID)) {
310            return false;
311        }
312        else {
313            Collection<String> availList = availableDeps.get(actionID);
314            if (!availList.removeAll(dependencyURIs)) {
315                return false;
316            }
317            synchronized (availList) {
318                if (availList.isEmpty()) {
319                    availableDeps.remove(actionID);
320                }
321            }
322        }
323        return true;
324    }
325
326    @Override
327    public void destroy() {
328        missingDeps.clear();
329        availableDeps.clear();
330    }
331
332    private static class SortedPKV {
333        private StringBuilder partKeys;
334        private StringBuilder partVals;
335
336        public SortedPKV(Map<String, String> partitions) {
337            this.partKeys = new StringBuilder();
338            this.partVals = new StringBuilder();
339            ArrayList<String> keys = new ArrayList<String>(partitions.keySet());
340            Collections.sort(keys);
341            for (String key : keys) {
342                this.partKeys.append(key).append(DELIMITER);
343                this.partVals.append(partitions.get(key)).append(DELIMITER);
344            }
345            this.partKeys.setLength(partKeys.length() - 1);
346            this.partVals.setLength(partVals.length() - 1);
347        }
348
349        public String getPartKeys() {
350            return partKeys.toString();
351        }
352
353        public String getPartVals() {
354            return partVals.toString();
355        }
356    }
357
358    private HCatURI removePartitions(String coordActionId, Collection<String> partKeys,
359            Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns) {
360        HCatURI hcatUri = null;
361        for (String partKey : partKeys) {
362            Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
363            Iterator<String> partValItr = partValues.keySet().iterator();
364            while (partValItr.hasNext()) {
365                String partVal = partValItr.next();
366                Collection<WaitingAction> waitingActions = partValues.get(partVal);
367                if (waitingActions != null) {
368                    Iterator<WaitingAction> waitItr = waitingActions.iterator();
369                    while (waitItr.hasNext()) {
370                        WaitingAction waction = waitItr.next();
371                        if (coordActionId.contains(waction.getActionID())) {
372                            waitItr.remove();
373                            if (hcatUri == null) {
374                                try {
375                                    hcatUri = new HCatURI(waction.getDependencyURI());
376                                }
377                                catch (URISyntaxException e) {
378                                    continue;
379                                }
380                            }
381                        }
382                    }
383                }
384                // delete partition value with no waiting actions
385                if (waitingActions.size() == 0) {
386                    partValItr.remove();
387                }
388            }
389            if (partValues.size() == 0) {
390                partKeyPatterns.remove(partKey);
391            }
392        }
393        return hcatUri;
394    }
395
396    @Override
397    public void removeNonWaitingCoordActions(Set<String> coordActions) {
398        HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
399        for (String coordActionId : coordActions) {
400            LOG.info("Removing non waiting coord action {0} from partition dependency map", coordActionId);
401            synchronized (actionPartitionMap) {
402                Map<String, Collection<String>> partitionMap = actionPartitionMap.get(coordActionId);
403                if (partitionMap != null) {
404                    Iterator<String> tableItr = partitionMap.keySet().iterator();
405                    while (tableItr.hasNext()) {
406                        String tableKey = tableItr.next();
407                        HCatURI hcatUri = null;
408                        Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
409                        if (partKeyPatterns != null) {
410                            synchronized (partKeyPatterns) {
411                                Collection<String> partKeys = partitionMap.get(tableKey);
412                                if (partKeys != null) {
413                                    hcatUri = removePartitions(coordActionId, partKeys, partKeyPatterns);
414                                }
415                            }
416                            if (partKeyPatterns.size() == 0) {
417                                tableItr.remove();
418                                if (hcatUri != null) {
419                                    // Close JMS session. Stop listening on topic
420                                    hcatService.unregisterFromNotification(hcatUri);
421                                }
422                            }
423                        }
424                    }
425                }
426                actionPartitionMap.remove(coordActionId);
427            }
428        }
429    }
430
431    @Override
432    public void removeCoordActionWithDependenciesAvailable(String coordAction) {
433        actionPartitionMap.remove(coordAction);
434    }
435
436    public String canonicalizeHostname(String name) {
437        return canonicalizeHostname(name, useCanonicalHostName);
438    }
439
440    public static String canonicalizeHostname(String name, boolean useCanonicalHostName) {
441        if (useCanonicalHostName) {
442            String hostname = name;
443            String port = null;
444            if (name.contains(":")) {
445                hostname = name.split(":")[0];
446                port = name.split(":")[1];
447            }
448            try {
449                InetAddress address = InetAddress.getByName(hostname);
450                String canonicalHostName = address.getCanonicalHostName();
451                if (null != port) {
452                    return canonicalHostName + ":" + port;
453                }
454                return canonicalHostName;
455            }
456            catch (IOException ex) {
457                LOG.error(ex);
458                return name;
459            }
460        }
461        else {
462            return name;
463        }
464    }
465}